Merge branch '5.0.0-beta' into develop

# Conflicts:
#	acl/pom.xml
#	broker/pom.xml
#	broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
#	broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
#	broker/src/main/java/org/apache/rocketmq/broker/plugin/MessageStoreFactory.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
#	broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java
#	broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
#	client/pom.xml
#	client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java
#	client/src/main/java/org/apache/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java
#	client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
#	client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
#	client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
#	client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java
#	common/pom.xml
#	common/src/main/java/org/apache/rocketmq/common/MQVersion.java
#	common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
#	common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
#	common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java
#	common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
#	common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
#	common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java
#	common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java
#	distribution/pom.xml
#	example/pom.xml
#	filter/pom.xml
#	logging/pom.xml
#	logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java
#	namesrv/pom.xml
#	namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java
#	namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java
#	namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java
#	openmessaging/pom.xml
#	pom.xml
#	remoting/pom.xml
#	remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java
#	remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java
#	remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java
#	remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingSerializable.java
#	srvutil/pom.xml
#	store/pom.xml
#	store/src/main/java/org/apache/rocketmq/store/CommitLog.java
#	store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
#	store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
#	store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
#	store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java
#	store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java
#	store/src/main/java/org/apache/rocketmq/store/ha/HAService.java
#	store/src/main/java/org/apache/rocketmq/store/index/IndexFile.java
#	store/src/main/java/org/apache/rocketmq/store/logfile/DefaultMappedFile.java
#	store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java
#	store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreCleanFilesTest.java
#	store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
#	store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java
#	test/pom.xml
#	tools/pom.xml
#	tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java
#	tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java
#	tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java
#	tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java
#	tools/src/main/java/org/apache/rocketmq/tools/command/topic/DeleteTopicSubCommand.java

.travis.yml

@@ -7,7 +7,7 @@ notifications:
     if: branch = develop OR branch = master
   on_success: change
   on_failure: always
-  
+
 
 language: java
 
@@ -48,10 +48,11 @@ before_script:
   - ulimit -c unlimited
 
 script:
-  - mvn -B verify -DskipTests
+  - mvn verify -DskipTests
   - travis_retry mvn -B clean apache-rat:check
-  - travis_retry mvn -B install jacoco:report coveralls:report
-  - travis_retry mvn -B clean install -pl test -Pit-test
+  - travis_retry mvn -B clean test jacoco:report coveralls:report
+  - travis_retry mvn -B clean test -pl test -Pit-test
+  - travis_retry mvn -B clean install -DskipTests
 
 after_success:
   - mvn sonar:sonar -Psonar-apache

acl/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.5-SNAPSHOT</version>
+        <version>5.0.0-BETA-SNAPSHOT</version>
     </parent>
     <artifactId>rocketmq-acl</artifactId>
     <name>rocketmq-acl ${project.version}</name>

acl/src/main/java/org/apache/rocketmq/acl/common/AclUtils.java

@@ -20,9 +20,11 @@ import com.alibaba.fastjson.JSONObject;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.nio.channels.FileChannel;
 import java.util.Map;
 import java.util.SortedMap;
 import org.apache.commons.lang3.StringUtils;
@@ -274,6 +276,43 @@ public class AclUtils {
         return true;
     }
 
+    public static boolean copyFile(String from, String to) {
+        FileChannel input = null;
+        FileChannel output = null;
+        try {
+            input = new FileInputStream(new File(from)).getChannel();
+            output = new FileOutputStream(new File(to)).getChannel();
+            output.transferFrom(input, 0, input.size());
+            return true;
+        } catch (Exception e) {
+            log.error("file copy error. from={}, to={}", from, to, e);
+        } finally {
+            closeFileChannel(input);
+            closeFileChannel(output);
+        }
+        return false;
+    }
+
+    public static boolean moveFile(String from, String to) {
+        try {
+            File file = new File(from);
+            return file.renameTo(new File(to));
+        } catch (Exception e) {
+            log.error("file move error. from={}, to={}", from, to, e);
+        }
+        return false;
+    }
+
+    private static void closeFileChannel(FileChannel fileChannel) {
+        if (fileChannel != null) {
+            try {
+                fileChannel.close();
+            } catch (IOException e) {
+                log.error("Close file channel error.", e);
+            }
+        }
+    }
+
     public static RPCHook getAclRPCHook(String fileName) {
         JSONObject yamlDataObject = null;
         try {

acl/src/test/java/org/apache/rocketmq/acl/common/AclUtilsTest.java

@@ -311,4 +311,25 @@ public class AclUtilsTest {
         Assert.assertNull(incompleteContRPCHook);
     }
 
+    @Test
+    public void testCopyAndMoveFile() {
+        String path = "src/test/resources/conf/plain_acl.yml";
+        String backupPath = "src/test/resources/conf/plain_acl.yml_backup";
+        Map<String, Object> aclYamlData = AclUtils.getYamlDataObject(path, Map.class);
+
+        AclUtils.copyFile(path, backupPath);
+        Assert.assertEquals(aclYamlData, AclUtils.getYamlDataObject(backupPath, Map.class));
+
+        Map<String, Object> aclYamlMap = new HashMap<String, Object>();
+        List<String> globalWhiteRemoteAddrs = new ArrayList<String>();
+        globalWhiteRemoteAddrs.add("10.10.103.*");
+        globalWhiteRemoteAddrs.add("192.168.0.*");
+        aclYamlMap.put("globalWhiteRemoteAddrs", globalWhiteRemoteAddrs);
+        AclUtils.writeDataObject(path, aclYamlMap);
+        Assert.assertNotEquals(aclYamlData, AclUtils.getYamlDataObject(path, Map.class));
+
+        AclUtils.moveFile(backupPath, path);
+        Assert.assertEquals(aclYamlData, AclUtils.getYamlDataObject(path, Map.class));
+    }
+
 }

acl/src/test/resources/conf/plain_acl.yml

@@ -18,7 +18,6 @@
 globalWhiteRemoteAddresses:
 - 10.10.103.*
 - 192.168.0.*
-
 accounts:
 - accessKey: RocketMQ
   secretKey: 12345678
@@ -31,14 +30,10 @@ accounts:
   - topicB=PUB|SUB
   - topicC=SUB
   groupPerms:
-  # the group should convert to retry topic
   - groupA=DENY
   - groupB=SUB
   - groupC=SUB
-
 - accessKey: rocketmq2
   secretKey: 12345678
   whiteRemoteAddress: 192.168.1.*
-  # if it is admin, it could access all resources
   admin: true
-

broker/pom.xml

@@ -13,7 +13,7 @@
     <parent>
         <groupId>org.apache.rocketmq</groupId>
         <artifactId>rocketmq-all</artifactId>
-        <version>4.9.5-SNAPSHOT</version>
+        <version>5.0.0-BETA-SNAPSHOT</version>
     </parent>
 
     <modelVersion>4.0.0</modelVersion>
@@ -50,6 +50,10 @@
             <groupId>${project.groupId}</groupId>
             <artifactId>rocketmq-acl</artifactId>
         </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
@@ -70,6 +74,10 @@
             <groupId>org.bouncycastle</groupId>
             <artifactId>bcpkix-jdk15on</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.googlecode.concurrentlinkedhashmap</groupId>
+            <artifactId>concurrentlinkedhashmap-lru</artifactId>
+        </dependency>
     </dependencies>
 
     <build>

broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java

@@ -35,6 +35,10 @@ public class BrokerPathConfigHelper {
         return rootDir + File.separator + "config" + File.separator + "topics.json";
     }
 
+    public static String getTopicQueueMappingPath(final String rootDir) {
+        return rootDir + File.separator + "config" + File.separator + "topicQueueMapping.json";
+    }
+
     public static String getConsumerOffsetPath(final String rootDir) {
         return rootDir + File.separator + "config" + File.separator + "consumerOffset.json";
     }
@@ -43,6 +47,10 @@ public class BrokerPathConfigHelper {
         return rootDir + File.separator + "config" + File.separator + "lmqConsumerOffset.json";
     }
 
+    public static String getConsumerOrderInfoPath(final String rootDir) {
+        return rootDir + File.separator + "config" + File.separator + "consumerOrderInfo.json";
+    }
+
     public static String getSubscriptionGroupPath(final String rootDir) {
         return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json";
     }
@@ -50,4 +58,8 @@ public class BrokerPathConfigHelper {
     public static String getConsumerFilterPath(final String rootDir) {
         return rootDir + File.separator + "config" + File.separator + "consumerFilter.json";
     }
+
+    public static String getMessageRequestModePath(final String rootDir) {
+        return rootDir + File.separator + "config" + File.separator + "messageRequestMode.json";
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/BrokerPreOnlineService.java

@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+import org.apache.rocketmq.broker.plugin.BrokerAttachedPlugin;
+import org.apache.rocketmq.common.BrokerSyncInfo;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.broker.schedule.DelayOffsetSerializeWrapper;
+import org.apache.rocketmq.store.ha.HAConnectionState;
+import org.apache.rocketmq.store.ha.HAConnectionStateNotificationRequest;
+
+public class BrokerPreOnlineService extends ServiceThread {
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final BrokerController brokerController;
+
+    private int waitBrokerIndex = 0;
+
+    public BrokerPreOnlineService(BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    @Override
+    public String getServiceName() {
+        if (this.brokerController != null && this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return brokerController.getBrokerIdentity().getLoggerIdentifier() + BrokerPreOnlineService.class.getSimpleName();
+        }
+        return BrokerPreOnlineService.class.getSimpleName();
+    }
+
+    @Override
+    public void run() {
+        LOGGER.info(this.getServiceName() + " service started");
+
+        while (!this.isStopped()) {
+            if (!this.brokerController.isIsolated()) {
+                LOGGER.info("broker {} is online", this.brokerController.getBrokerConfig().getCanonicalName());
+                break;
+            }
+            try {
+                boolean isSuccess = this.prepareForBrokerOnline();
+                if (!isSuccess) {
+                    this.waitForRunning(1000);
+                } else {
+                    break;
+                }
+            } catch (Exception e) {
+                LOGGER.error("Broker preOnline error, ", e);
+            }
+        }
+
+        LOGGER.info(this.getServiceName() + " service end");
+    }
+
+    CompletableFuture<Boolean> waitForHaHandshakeComplete(String brokerAddr) {
+        LOGGER.info("wait for handshake completion with {}", brokerAddr);
+        HAConnectionStateNotificationRequest request =
+            new HAConnectionStateNotificationRequest(HAConnectionState.TRANSFER, RemotingHelper.parseHostFromAddress(brokerAddr), true);
+        if (this.brokerController.getMessageStore().getHaService() != null) {
+            this.brokerController.getMessageStore().getHaService().putGroupConnectionStateRequest(request);
+        } else {
+            LOGGER.error("HAService is null, maybe broker config is wrong. For example, duplicationEnable is true");
+            request.getRequestFuture().complete(false);
+        }
+        return request.getRequestFuture();
+    }
+
+    private boolean futureWaitAction(boolean result, BrokerMemberGroup brokerMemberGroup) {
+        if (!result) {
+            LOGGER.error("wait for handshake completion failed, HA connection lost");
+            return false;
+        }
+        if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID) {
+            LOGGER.info("slave preOnline complete, start service");
+            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+            this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+        }
+        return true;
+    }
+
+    private boolean prepareForMasterOnline(BrokerMemberGroup brokerMemberGroup) {
+        List<Long> brokerIdList = new ArrayList<>(brokerMemberGroup.getBrokerAddrs().keySet());
+        Collections.sort(brokerIdList);
+        while (true) {
+            if (waitBrokerIndex >= brokerIdList.size()) {
+                LOGGER.info("master preOnline complete, start service");
+                this.brokerController.startService(MixAll.MASTER_ID, this.brokerController.getBrokerAddr());
+                return true;
+            }
+
+            String brokerAddrToWait = brokerMemberGroup.getBrokerAddrs().get(brokerIdList.get(waitBrokerIndex));
+
+            try {
+                this.brokerController.getBrokerOuterAPI().
+                    sendBrokerHaInfo(brokerAddrToWait, this.brokerController.getHAServerAddr(),
+                        this.brokerController.getMessageStore().getBrokerInitMaxOffset(), this.brokerController.getBrokerAddr());
+            } catch (Exception e) {
+                LOGGER.error("send ha address to {} exception, {}", brokerAddrToWait, e);
+                return false;
+            }
+
+            CompletableFuture<Boolean> haHandshakeFuture = waitForHaHandshakeComplete(brokerAddrToWait)
+                .thenApply(result -> futureWaitAction(result, brokerMemberGroup));
+
+            try {
+                if (!haHandshakeFuture.get()) {
+                    return false;
+                }
+            } catch (Exception e) {
+                LOGGER.error("Wait handshake completion exception, {}", e);
+                return false;
+            }
+
+            if (syncMetadataReverse(brokerAddrToWait)) {
+                waitBrokerIndex++;
+            } else {
+                return false;
+            }
+        }
+    }
+
+    private boolean syncMetadataReverse(String brokerAddr) {
+        try {
+            LOGGER.info("Get metadata reverse from {}", brokerAddr);
+
+            String delayOffset = this.brokerController.getBrokerOuterAPI().getAllDelayOffset(brokerAddr);
+            DelayOffsetSerializeWrapper delayOffsetSerializeWrapper =
+                DelayOffsetSerializeWrapper.fromJson(delayOffset, DelayOffsetSerializeWrapper.class);
+
+            ConsumerOffsetSerializeWrapper consumerOffsetSerializeWrapper = this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(brokerAddr);
+
+            if (null != consumerOffsetSerializeWrapper && brokerController.getConsumerOffsetManager().getDataVersion().compare(consumerOffsetSerializeWrapper.getDataVersion()) <= 0) {
+                LOGGER.info("{}'s consumerOffset data version is larger than master broker, {}'s consumerOffset will be used.", brokerAddr, brokerAddr);
+                this.brokerController.getConsumerOffsetManager().getOffsetTable()
+                    .putAll(consumerOffsetSerializeWrapper.getOffsetTable());
+                this.brokerController.getConsumerOffsetManager().getDataVersion().assignNewOne(consumerOffsetSerializeWrapper.getDataVersion());
+                this.brokerController.getConsumerOffsetManager().persist();
+            }
+
+            if (null != delayOffset && brokerController.getScheduleMessageService().getDataVersion().compare(delayOffsetSerializeWrapper.getDataVersion()) <= 0) {
+                LOGGER.info("{}'s scheduleMessageService data version is larger than master broker, {}'s delayOffset will be used.", brokerAddr, brokerAddr);
+                String fileName =
+                    StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController
+                        .getMessageStoreConfig().getStorePathRootDir());
+                try {
+                    MixAll.string2File(delayOffset, fileName);
+                    this.brokerController.getScheduleMessageService().load();
+                } catch (IOException e) {
+                    LOGGER.error("Persist file Exception, {}", fileName, e);
+                }
+            }
+
+            for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {
+                if (brokerAttachedPlugin != null) {
+                    brokerAttachedPlugin.syncMetadataReverse(brokerAddr);
+                }
+            }
+
+        } catch (Exception e) {
+            LOGGER.error("GetMetadataReverse Failed", e);
+            return false;
+        }
+
+        return true;
+    }
+
+    private boolean prepareForSlaveOnline(BrokerMemberGroup brokerMemberGroup) {
+        BrokerSyncInfo brokerSyncInfo;
+        try {
+            brokerSyncInfo = this.brokerController.getBrokerOuterAPI()
+                .retrieveBrokerHaInfo(brokerMemberGroup.getBrokerAddrs().get(MixAll.MASTER_ID));
+        } catch (Exception e) {
+            LOGGER.error("retrieve master ha info exception, {}", e);
+            return false;
+        }
+
+        if (this.brokerController.getMessageStore().getMasterFlushedOffset() == 0
+            && this.brokerController.getMessageStoreConfig().isSyncMasterFlushOffsetWhenStartup()) {
+            LOGGER.info("Set master flush offset in slave to {}", brokerSyncInfo.getMasterFlushOffset());
+            this.brokerController.getMessageStore().setMasterFlushedOffset(brokerSyncInfo.getMasterFlushOffset());
+        }
+
+        if (brokerSyncInfo.getMasterHaAddress() != null) {
+            this.brokerController.getMessageStore().updateHaMasterAddress(brokerSyncInfo.getMasterHaAddress());
+            this.brokerController.getMessageStore().updateMasterAddress(brokerSyncInfo.getMasterAddress());
+        } else {
+            LOGGER.info("fetch master ha address return null, start service directly");
+            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+            this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+            return true;
+        }
+
+        CompletableFuture<Boolean> haHandshakeFuture = waitForHaHandshakeComplete(brokerSyncInfo.getMasterHaAddress())
+            .thenApply(result -> futureWaitAction(result, brokerMemberGroup));
+
+        try {
+            if (!haHandshakeFuture.get()) {
+                return false;
+            }
+        } catch (Exception e) {
+            LOGGER.error("Wait handshake completion exception, {}", e);
+            return false;
+        }
+
+        return true;
+    }
+
+    private boolean prepareForBrokerOnline() {
+        BrokerMemberGroup brokerMemberGroup;
+        try {
+            brokerMemberGroup = this.brokerController.getBrokerOuterAPI().syncBrokerMemberGroup(
+                this.brokerController.getBrokerConfig().getBrokerClusterName(),
+                this.brokerController.getBrokerConfig().getBrokerName(),
+                this.brokerController.getBrokerConfig().isCompatibleWithOldNameSrv());
+        } catch (Exception e) {
+            LOGGER.error("syncBrokerMemberGroup from namesrv error, start service failed, will try later, ", e);
+            return false;
+        }
+
+        if (brokerMemberGroup != null && !brokerMemberGroup.getBrokerAddrs().isEmpty()) {
+            long minBrokerId = getMinBrokerId(brokerMemberGroup.getBrokerAddrs());
+
+            if (this.brokerController.getBrokerConfig().getBrokerId() == MixAll.MASTER_ID) {
+                return prepareForMasterOnline(brokerMemberGroup);
+            } else if (minBrokerId == MixAll.MASTER_ID) {
+                return prepareForSlaveOnline(brokerMemberGroup);
+            } else {
+                LOGGER.info("no master online, start service directly");
+                this.brokerController.startService(minBrokerId, brokerMemberGroup.getBrokerAddrs().get(minBrokerId));
+            }
+        } else {
+            LOGGER.info("no other broker online, will start service directly");
+            this.brokerController.startService(this.brokerController.getBrokerConfig().getBrokerId(), this.brokerController.getBrokerAddr());
+        }
+
+        return true;
+    }
+
+    private long getMinBrokerId(Map<Long, String> brokerAddrMap) {
+        Map<Long, String> brokerAddrMapCopy = new HashMap<>(brokerAddrMap);
+        brokerAddrMapCopy.remove(this.brokerController.getBrokerConfig().getBrokerId());
+        if (!brokerAddrMapCopy.isEmpty()) {
+            return Collections.min(brokerAddrMapCopy.keySet());
+        }
+        return this.brokerController.getBrokerConfig().getBrokerId();
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java

@@ -19,9 +19,9 @@ package org.apache.rocketmq.broker;
 import ch.qos.logback.classic.LoggerContext;
 import ch.qos.logback.classic.joran.JoranConfigurator;
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
@@ -37,6 +37,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.srvutil.ServerUtil;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedInputStream;
@@ -52,6 +53,7 @@ public class BrokerStartup {
     public static CommandLine commandLine = null;
     public static String configFile = null;
     public static InternalLogger log;
+    public static SystemConfigFileHelper configFileHelper = new SystemConfigFileHelper();
 
     public static void main(String[] args) {
         start(createBrokerController(args));
@@ -93,7 +95,7 @@ public class BrokerStartup {
             //PackageConflictDetect.detectFastjson();
             Options options = ServerUtil.buildCommandlineOptions(new Options());
             commandLine = ServerUtil.parseCmdLine("mqbroker", args, buildCommandlineOptions(options),
-                new PosixParser());
+                new DefaultParser());
             if (null == commandLine) {
                 System.exit(-1);
             }
@@ -115,22 +117,21 @@ public class BrokerStartup {
             if (commandLine.hasOption('c')) {
                 String file = commandLine.getOptionValue('c');
                 if (file != null) {
+                    configFileHelper.setFile(file);
                     configFile = file;
-                    InputStream in = new BufferedInputStream(new FileInputStream(file));
-                    properties = new Properties();
-                    properties.load(in);
-
-                    properties2SystemEnv(properties);
-                    MixAll.properties2Object(properties, brokerConfig);
-                    MixAll.properties2Object(properties, nettyServerConfig);
-                    MixAll.properties2Object(properties, nettyClientConfig);
-                    MixAll.properties2Object(properties, messageStoreConfig);
-
                     BrokerPathConfigHelper.setBrokerConfigPath(file);
-                    in.close();
                 }
             }
 
+            properties = configFileHelper.loadConfig();
+            if (properties != null) {
+                properties2SystemEnv(properties);
+                MixAll.properties2Object(properties, brokerConfig);
+                MixAll.properties2Object(properties, nettyServerConfig);
+                MixAll.properties2Object(properties, nettyClientConfig);
+                MixAll.properties2Object(properties, messageStoreConfig);
+            }
+
             MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig);
 
             if (null == brokerConfig.getRocketmqHome()) {
@@ -209,6 +210,8 @@ public class BrokerStartup {
             MixAll.printObjectProperties(log, nettyClientConfig);
             MixAll.printObjectProperties(log, messageStoreConfig);
 
+            brokerConfig.setInBrokerContainer(false);
+
             final BrokerController controller = new BrokerController(
                 brokerConfig,
                 nettyServerConfig,
@@ -276,4 +279,33 @@ public class BrokerStartup {
 
         return options;
     }
+
+    public static class SystemConfigFileHelper {
+        private static final Logger LOGGER = LoggerFactory.getLogger(SystemConfigFileHelper.class);
+
+        private String file;
+
+        public SystemConfigFileHelper() {
+        }
+
+        public Properties loadConfig() throws Exception {
+            InputStream in = new BufferedInputStream(new FileInputStream(file));
+            Properties properties = new Properties();
+            properties.load(in);
+            in.close();
+            return properties;
+        }
+
+        public void update(Properties properties) throws Exception {
+            LOGGER.error("[SystemConfigFileHelper] update no thing.");
+        }
+
+        public void setFile(String file) {
+            this.file = file;
+        }
+
+        public String getFile() {
+            return file;
+        }
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/ShutdownHook.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker;
+
+public interface ShutdownHook {
+    /**
+     * Code to execute before broker shutdown.
+     *
+     * @param controller broker to shutdown
+     */
+    void beforeShutdown(BrokerController controller);
+}

broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java

@@ -17,8 +17,8 @@
 package org.apache.rocketmq.broker.client;
 
 import io.netty.channel.Channel;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
@@ -31,11 +31,12 @@ public class ClientHousekeepingService implements ChannelEventListener {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
 
-    private ScheduledExecutorService scheduledExecutorService = Executors
-        .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ClientHousekeepingScheduledThread"));
+    private ScheduledExecutorService scheduledExecutorService;
 
     public ClientHousekeepingService(final BrokerController brokerController) {
         this.brokerController = brokerController;
+        scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
+            new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity()));
     }
 
     public void start() {
@@ -64,7 +65,7 @@ public class ClientHousekeepingService implements ChannelEventListener {
 
     @Override
     public void onChannelConnect(String remoteAddr, Channel channel) {
-
+        this.brokerController.getBrokerStatsManager().incChannelConnectNum();
     }
 
     @Override
@@ -72,6 +73,7 @@ public class ClientHousekeepingService implements ChannelEventListener {
         this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel);
+        this.brokerController.getBrokerStatsManager().incChannelCloseNum();
     }
 
     @Override
@@ -79,6 +81,7 @@ public class ClientHousekeepingService implements ChannelEventListener {
         this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel);
+        this.brokerController.getBrokerStatsManager().incChannelExceptionNum();
     }
 
     @Override
@@ -86,5 +89,6 @@ public class ClientHousekeepingService implements ChannelEventListener {
         this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel);
         this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel);
+        this.brokerController.getBrokerStatsManager().incChannelIdleNum();
     }
 }

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java

@@ -68,6 +68,10 @@ public class ConsumerGroupInfo {
         return subscriptionTable;
     }
 
+    public ClientChannelInfo findChannel(final Channel channel) {
+        return this.channelInfoTable.get(channel);
+    }
+
     public ConcurrentMap<Channel, ClientChannelInfo> getChannelInfoTable() {
         return channelInfoTable;
     }
@@ -113,6 +117,15 @@ public class ConsumerGroupInfo {
         return false;
     }
 
+    /**
+     * Update {@link #channelInfoTable} in {@link ConsumerGroupInfo}
+     *
+     * @param infoNew Channel info of new client.
+     * @param consumeType consume type of new client.
+     * @param messageModel message consuming model (CLUSTERING/BROADCASTING) of new client.
+     * @param consumeFromWhere indicate the position when the client consume message firstly.
+     * @return the result that if new connector is connected or not.
+     */
     public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType,
         MessageModel messageModel, ConsumeFromWhere consumeFromWhere) {
         boolean updated = false;
@@ -132,9 +145,9 @@ public class ConsumerGroupInfo {
             infoOld = infoNew;
         } else {
             if (!infoOld.getClientId().equals(infoNew.getClientId())) {
-                log.error("[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ",
-                    this.groupName,
-                    infoOld.toString(),
+                log.error(
+                    "ConsumerGroupInfo: consumer channel exists in broker, but clientId is not the same one, "
+                        + "group={}, old clientChannelInfo={}, new clientChannelInfo={}", groupName, infoOld.toString(),
                     infoNew.toString());
                 this.channelInfoTable.put(infoNew.getChannel(), infoNew);
             }
@@ -146,6 +159,12 @@ public class ConsumerGroupInfo {
         return updated;
     }
 
+    /**
+     * Update subscription.
+     *
+     * @param subList set of {@link SubscriptionData}
+     * @return the boolean indicates the subscription has changed or not.
+     */
     public boolean updateSubscription(final Set<SubscriptionData> subList) {
         boolean updated = false;
 

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerIdsChangeListener.java

@@ -19,4 +19,6 @@ package org.apache.rocketmq.broker.client;
 public interface ConsumerIdsChangeListener {
 
     void handle(ConsumerGroupEvent event, String group, Object... args);
+
+    void shutdown();
 }

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java

@@ -16,32 +16,41 @@
  */
 package org.apache.rocketmq.broker.client;
 
-import io.netty.channel.Channel;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+
+import io.netty.channel.Channel;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
 import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
 import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class ConsumerManager {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
-    private final ConcurrentMap<String/* Group */, ConsumerGroupInfo> consumerTable =
+    private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =
         new ConcurrentHashMap<String, ConsumerGroupInfo>(1024);
     private final ConsumerIdsChangeListener consumerIdsChangeListener;
+    protected final BrokerStatsManager brokerStatsManager;
 
     public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener) {
         this.consumerIdsChangeListener = consumerIdsChangeListener;
+        this.brokerStatsManager = null;
+    }
+
+    public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, final BrokerStatsManager brokerStatsManager) {
+        this.consumerIdsChangeListener = consumerIdsChangeListener;
+        this.brokerStatsManager = brokerStatsManager;
     }
 
     public ClientChannelInfo findChannel(final String group, final String clientId) {
@@ -61,6 +70,10 @@ public class ConsumerManager {
         return null;
     }
 
+    public ConcurrentMap<String, ConsumerGroupInfo> getConsumerTable() {
+        return this.consumerTable;
+    }
+
     public ConsumerGroupInfo getConsumerGroupInfo(final String group) {
         return this.consumerTable.get(group);
     }
@@ -74,17 +87,18 @@ public class ConsumerManager {
         return 0;
     }
 
-    public void doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+    public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+        boolean removed = false;
         Iterator<Entry<String, ConsumerGroupInfo>> it = this.consumerTable.entrySet().iterator();
         while (it.hasNext()) {
             Entry<String, ConsumerGroupInfo> next = it.next();
             ConsumerGroupInfo info = next.getValue();
-            boolean removed = info.doChannelCloseEvent(remoteAddr, channel);
+            removed = info.doChannelCloseEvent(remoteAddr, channel);
             if (removed) {
                 if (info.getChannelInfoTable().isEmpty()) {
                     ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey());
                     if (remove != null) {
-                        log.info("unregister consumer ok, no any connection, and remove consumer group, {}",
+                        LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}",
                             next.getKey());
                         this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, next.getKey());
                     }
@@ -93,12 +107,13 @@ public class ConsumerManager {
                 this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
             }
         }
+        return removed;
     }
 
     public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
         ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
         final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {
-
+        long start = System.currentTimeMillis();
         ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
         if (null == consumerGroupInfo) {
             ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
@@ -116,6 +131,9 @@ public class ConsumerManager {
                 this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
             }
         }
+        if (null != this.brokerStatsManager) {
+            this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));
+        }
 
         this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);
 
@@ -130,7 +148,7 @@ public class ConsumerManager {
             if (consumerGroupInfo.getChannelInfoTable().isEmpty()) {
                 ConsumerGroupInfo remove = this.consumerTable.remove(group);
                 if (remove != null) {
-                    log.info("unregister consumer ok, no any connection, and remove consumer group, {}", group);
+                    LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}", group);
 
                     this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, group);
                 }
@@ -156,7 +174,7 @@ public class ConsumerManager {
                 ClientChannelInfo clientChannelInfo = nextChannel.getValue();
                 long diff = System.currentTimeMillis() - clientChannelInfo.getLastUpdateTimestamp();
                 if (diff > CHANNEL_EXPIRED_TIMEOUT) {
-                    log.warn(
+                    LOGGER.warn(
                         "SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}",
                         RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group);
                     RemotingUtil.closeChannel(clientChannelInfo.getChannel());
@@ -165,7 +183,7 @@ public class ConsumerManager {
             }
 
             if (channelInfoTable.isEmpty()) {
-                log.warn(
+                LOGGER.warn(
                     "SCAN: remove expired channel from ConsumerManager consumerTable, all clear, consumerGroup={}",
                     group);
                 it.remove();

broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java

@@ -17,18 +17,46 @@
 package org.apache.rocketmq.broker.client;
 
 import io.netty.channel.Channel;
-
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
+import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.utils.ThreadUtils;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
 
 public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
+    private final int cacheSize = 8096;
+
+    private final ScheduledExecutorService scheduledExecutorService =  new ScheduledThreadPoolExecutor(1,
+        ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true));
+
+    private ConcurrentHashMap<String,List<Channel>> consumerChannelMap = new ConcurrentHashMap<>(cacheSize);
 
     public DefaultConsumerIdsChangeListener(BrokerController brokerController) {
         this.brokerController = brokerController;
+
+        scheduledExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(brokerController.getBrokerConfig()) {
+            @Override
+            public void run2() {
+                try {
+                    notifyConsumerChange();
+                } catch (Exception e) {
+                    log.error(
+                        "DefaultConsumerIdsChangeListen#notifyConsumerChange: unexpected error occurs", e);
+                }
+            }
+        }, 30, 15, TimeUnit.SECONDS);
     }
 
     @Override
@@ -43,8 +71,12 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
                 }
                 List<Channel> channels = (List<Channel>) args[0];
                 if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
-                    for (Channel chl : channels) {
-                        this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
+                    if (this.brokerController.getBrokerConfig().isRealTimeNotifyConsumerChange()) {
+                        for (Channel chl : channels) {
+                            this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group);
+                        }
+                    } else {
+                        consumerChannelMap.put(group, channels);
                     }
                 }
                 break;
@@ -62,4 +94,34 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
                 throw new RuntimeException("Unknown event " + event);
         }
     }
+
+    private void notifyConsumerChange() {
+
+        if (consumerChannelMap.isEmpty()) {
+            return;
+        }
+
+        ConcurrentHashMap<String, List<Channel>> processMap = new ConcurrentHashMap<>(consumerChannelMap);
+        consumerChannelMap = new ConcurrentHashMap<>(cacheSize);
+
+        for (Map.Entry<String, List<Channel>> entry : processMap.entrySet()) {
+            String consumerId = entry.getKey();
+            List<Channel> channelList = entry.getValue();
+            try {
+                if (channelList != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) {
+                    for (Channel chl : channelList) {
+                        this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, consumerId);
+                    }
+                }
+            } catch (Exception e) {
+                log.error("Failed to notify consumer when some consumers changed, consumerId to notify: {}",
+                    consumerId, e);
+            }
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        this.scheduledExecutorService.shutdown();
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java

@@ -33,6 +33,7 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class ProducerManager {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -41,9 +42,24 @@ public class ProducerManager {
     private final ConcurrentHashMap<String /* group name */, ConcurrentHashMap<Channel, ClientChannelInfo>> groupChannelTable =
         new ConcurrentHashMap<>();
     private final ConcurrentHashMap<String, Channel> clientChannelTable = new ConcurrentHashMap<>();
+    protected final BrokerStatsManager brokerStatsManager;
     private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();
 
     public ProducerManager() {
+        this.brokerStatsManager = null;
+    }
+
+    public ProducerManager(final BrokerStatsManager brokerStatsManager) {
+        this.brokerStatsManager = brokerStatsManager;
+    }
+
+    public int groupSize() {
+        return this.groupChannelTable.size();
+    }
+
+    public boolean groupOnline(String group) {
+        Map<Channel, ClientChannelInfo> channels = this.groupChannelTable.get(group);
+        return channels != null && !channels.isEmpty();
     }
 
     public ConcurrentHashMap<String, ConcurrentHashMap<Channel, ClientChannelInfo>> getGroupChannelTable() {
@@ -94,7 +110,7 @@ public class ProducerManager {
                     it.remove();
                     clientChannelTable.remove(info.getClientId());
                     log.warn(
-                            "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}",
+                            "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}",
                             RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group);
                     RemotingUtil.closeChannel(info.getChannel());
                 }
@@ -102,7 +118,8 @@ public class ProducerManager {
         }
     }
 
-    public synchronized void doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+    public synchronized boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+        boolean removed = false;
         if (channel != null) {
             for (final Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>> entry : this.groupChannelTable
                     .entrySet()) {
@@ -113,6 +130,7 @@ public class ProducerManager {
                         clientChannelInfoTable.remove(channel);
                 if (clientChannelInfo != null) {
                     clientChannelTable.remove(clientChannelInfo.getClientId());
+                    removed = true;
                     log.info(
                             "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}",
                             clientChannelInfo.toString(), remoteAddr, group);
@@ -120,6 +138,7 @@ public class ProducerManager {
 
             }
         }
+        return removed;
     }
 
     public synchronized void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) {

broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java

@@ -76,7 +76,7 @@ public class Broker2Client {
     }
 
     public RemotingCommand callClient(final Channel channel,
-                                      final RemotingCommand request
+        final RemotingCommand request
     ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException {
         return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000);
     }
@@ -101,12 +101,13 @@ public class Broker2Client {
         }
     }
 
+
     public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) {
         return resetOffset(topic, group, timeStamp, isForce, false);
     }
 
     public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce,
-                                       boolean isC) {
+        boolean isC) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(null);
 
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -237,8 +238,7 @@ public class Broker2Client {
             RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT,
                 requestHeader);
 
-        Map<String, Map<MessageQueue, Long>> consumerStatusTable =
-            new HashMap<String, Map<MessageQueue, Long>>();
+        Map<String, Map<MessageQueue, Long>> consumerStatusTable = new HashMap<String, Map<MessageQueue, Long>>();
         ConcurrentMap<Channel, ClientChannelInfo> channelInfoTable =
             this.brokerController.getConsumerManager().getConsumerGroupInfo(group).getChannelInfoTable();
         if (null == channelInfoTable || channelInfoTable.isEmpty()) {

broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java

@@ -16,16 +16,17 @@
  */
 package org.apache.rocketmq.broker.client.rebalance;
 
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
-import org.apache.rocketmq.common.message.MessageQueue;
 
 public class RebalanceLockManager {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME);
@@ -35,6 +36,19 @@ public class RebalanceLockManager {
     private final ConcurrentMap<String/* group */, ConcurrentHashMap<MessageQueue, LockEntry>> mqLockTable =
         new ConcurrentHashMap<String, ConcurrentHashMap<MessageQueue, LockEntry>>(1024);
 
+    public boolean isLockAllExpired(final String group) {
+        final ConcurrentHashMap<MessageQueue, LockEntry> lockEntryMap = mqLockTable.get(group);
+        if (null == lockEntryMap) {
+            return true;
+        }
+        for (LockEntry entry : lockEntryMap.values()) {
+            if (!entry.isExpired()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     public boolean tryLock(final String group, final MessageQueue mq, final String clientId) {
 
         if (!this.isLocked(group, mq, clientId)) {
@@ -52,10 +66,9 @@ public class RebalanceLockManager {
                         lockEntry = new LockEntry();
                         lockEntry.setClientId(clientId);
                         groupValue.put(mq, lockEntry);
-                        log.info("tryLock, message queue not locked, I got it. Group: {} NewClientId: {} {}",
-                            group,
-                            clientId,
-                            mq);
+                        log.info(
+                            "RebalanceLockManager#tryLock: lock a message queue which has not been locked yet, "
+                                + "group={}, clientId={}, mq={}", group, clientId, mq);
                     }
 
                     if (lockEntry.isLocked(clientId)) {
@@ -69,26 +82,21 @@ public class RebalanceLockManager {
                         lockEntry.setClientId(clientId);
                         lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());
                         log.warn(
-                            "tryLock, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}",
-                            group,
-                            oldClientId,
-                            clientId,
-                            mq);
+                            "RebalanceLockManager#tryLock: try to lock a expired message queue, group={}, mq={}, old "
+                                + "client id={}, new client id={}", group, mq, oldClientId, clientId);
                         return true;
                     }
 
                     log.warn(
-                        "tryLock, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}",
-                        group,
-                        oldClientId,
-                        clientId,
-                        mq);
+                        "RebalanceLockManager#tryLock: message queue has been locked by other client, group={}, "
+                            + "mq={}, locked client id={}, current client id={}", group, mq, oldClientId, clientId);
                     return false;
                 } finally {
                     this.lock.unlock();
                 }
             } catch (InterruptedException e) {
-                log.error("putMessage exception", e);
+                log.error("RebalanceLockManager#tryLock: unexpected error, group={}, mq={}, clientId={}", group, mq,
+                    clientId, e);
             }
         } else {
 
@@ -144,10 +152,8 @@ public class RebalanceLockManager {
                             lockEntry.setClientId(clientId);
                             groupValue.put(mq, lockEntry);
                             log.info(
-                                "tryLockBatch, message queue not locked, I got it. Group: {} NewClientId: {} {}",
-                                group,
-                                clientId,
-                                mq);
+                                "RebalanceLockManager#tryLockBatch: lock a message which has not been locked yet, "
+                                    + "group={}, clientId={}, mq={}", group, clientId, mq);
                         }
 
                         if (lockEntry.isLocked(clientId)) {
@@ -162,27 +168,23 @@ public class RebalanceLockManager {
                             lockEntry.setClientId(clientId);
                             lockEntry.setLastUpdateTimestamp(System.currentTimeMillis());
                             log.warn(
-                                "tryLockBatch, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}",
-                                group,
-                                oldClientId,
-                                clientId,
-                                mq);
+                                "RebalanceLockManager#tryLockBatch: try to lock a expired message queue, group={}, "
+                                    + "mq={}, old client id={}, new client id={}", group, mq, oldClientId, clientId);
                             lockedMqs.add(mq);
                             continue;
                         }
 
                         log.warn(
-                            "tryLockBatch, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}",
-                            group,
-                            oldClientId,
-                            clientId,
-                            mq);
+                            "RebalanceLockManager#tryLockBatch: message queue has been locked by other client, "
+                                + "group={}, mq={}, locked client id={}, current client id={}", group, mq, oldClientId,
+                            clientId);
                     }
                 } finally {
                     this.lock.unlock();
                 }
             } catch (InterruptedException e) {
-                log.error("putMessage exception", e);
+                log.error("RebalanceLockManager#tryBatch: unexpected error, group={}, mqs={}, clientId={}", group, mqs,
+                    clientId, e);
             }
         }
 
@@ -200,34 +202,29 @@ public class RebalanceLockManager {
                         if (null != lockEntry) {
                             if (lockEntry.getClientId().equals(clientId)) {
                                 groupValue.remove(mq);
-                                log.info("unlockBatch, Group: {} {} {}",
-                                    group,
-                                    mq,
-                                    clientId);
+                                log.info("RebalanceLockManager#unlockBatch: unlock mq, group={}, clientId={}, mqs={}",
+                                    group, clientId, mq);
                             } else {
-                                log.warn("unlockBatch, but mq locked by other client: {}, Group: {} {} {}",
-                                    lockEntry.getClientId(),
-                                    group,
-                                    mq,
-                                    clientId);
+                                log.warn(
+                                    "RebalanceLockManager#unlockBatch: mq locked by other client, group={}, locked "
+                                        + "clientId={}, current clientId={}, mqs={}", group, lockEntry.getClientId(),
+                                    clientId, mq);
                             }
                         } else {
-                            log.warn("unlockBatch, but mq not locked, Group: {} {} {}",
-                                group,
-                                mq,
-                                clientId);
+                            log.warn("RebalanceLockManager#unlockBatch: mq not locked, group={}, clientId={}, mq={}",
+                                group, clientId, mq);
                         }
                     }
                 } else {
-                    log.warn("unlockBatch, group not exist, Group: {} {}",
-                        group,
-                        clientId);
+                    log.warn("RebalanceLockManager#unlockBatch: group not exist, group={}, clientId={}, mqs={}", group,
+                        clientId, mqs);
                 }
             } finally {
                 this.lock.unlock();
             }
         } catch (InterruptedException e) {
-            log.error("putMessage exception", e);
+            log.error("RebalanceLockManager#unlockBatch: unexpected error, group={}, mqs={}, clientId={}", group, mqs,
+                clientId);
         }
     }
 

broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java

@@ -22,6 +22,8 @@ import io.openmessaging.storage.dledger.MemberState;
 import io.openmessaging.storage.dledger.utils.DLedgerUtils;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -33,17 +35,21 @@ import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
 
 public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {
 
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-    private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_"));
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private ExecutorService executorService;
     private BrokerController brokerController;
     private DefaultMessageStore messageStore;
     private DLedgerCommitLog dLedgerCommitLog;
     private DLedgerServer dLegerServer;
+    private Future<?> slaveSyncFuture;
+
     public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) {
         this.brokerController = brokerController;
         this.messageStore = messageStore;
         this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
         this.dLegerServer = dLedgerCommitLog.getdLedgerServer();
+        this.executorService = Executors.newSingleThreadExecutor(
+            new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity()));
     }
 
     @Override public void handle(long term, MemberState.Role role) {
@@ -52,15 +58,15 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange
                 long start = System.currentTimeMillis();
                 try {
                     boolean succ = true;
-                    log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole());
+                    LOGGER.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole());
                     switch (role) {
                         case CANDIDATE:
                             if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
-                                brokerController.changeToSlave(dLedgerCommitLog.getId());
+                                changeToSlave(dLedgerCommitLog.getId());
                             }
                             break;
                         case FOLLOWER:
-                            brokerController.changeToSlave(dLedgerCommitLog.getId());
+                            changeToSlave(dLedgerCommitLog.getId());
                             break;
                         case LEADER:
                             while (true) {
@@ -79,21 +85,89 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange
                             }
                             if (succ) {
                                 messageStore.recoverTopicQueueTable();
-                                brokerController.changeToMaster(BrokerRole.SYNC_MASTER);
+                                changeToMaster(BrokerRole.SYNC_MASTER);
                             }
                             break;
                         default:
                             break;
                     }
-                    log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start));
+                    LOGGER.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start));
                 } catch (Throwable t) {
-                    log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t);
+                    LOGGER.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t);
                 }
             }
         };
         executorService.submit(runnable);
     }
 
+    private void handleSlaveSynchronize(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.brokerController.getSlaveSynchronize().setMasterAddr(null);
+            slaveSyncFuture = this.brokerController.getScheduledExecutorService().scheduleAtFixedRate(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        brokerController.getSlaveSynchronize().syncAll();
+                    } catch (Throwable e) {
+                        LOGGER.error("ScheduledTask SlaveSynchronize syncAll error.", e);
+                    }
+                }
+            }, 1000 * 3, 1000 * 10, TimeUnit.MILLISECONDS);
+        } else {
+            //handle the slave synchronise
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.brokerController.getSlaveSynchronize().setMasterAddr(null);
+        }
+    }
+
+    public void changeToSlave(int brokerId) {
+        LOGGER.info("Begin to change to slave brokerName={} brokerId={}", this.brokerController.getBrokerConfig().getBrokerName(), brokerId);
+
+        //change the role
+        this.brokerController.getBrokerConfig().setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check
+        this.brokerController.getMessageStoreConfig().setBrokerRole(BrokerRole.SLAVE);
+
+        this.brokerController.changeSpecialServiceStatus(false);
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(BrokerRole.SLAVE);
+
+        try {
+            this.brokerController.registerBrokerAll(true, true, this.brokerController.getBrokerConfig().isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        LOGGER.info("Finish to change to slave brokerName={} brokerId={}", this.brokerController.getBrokerConfig().getBrokerName(), brokerId);
+    }
+
+    public void changeToMaster(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            return;
+        }
+        LOGGER.info("Begin to change to master brokerName={}", this.brokerController.getBrokerConfig().getBrokerName());
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(role);
+
+        this.brokerController.changeSpecialServiceStatus(true);
+
+        //if the operations above are totally successful, we change to master
+        this.brokerController.getBrokerConfig().setBrokerId(0); //TO DO check
+        this.brokerController.getMessageStoreConfig().setBrokerRole(role);
+
+        try {
+            this.brokerController.registerBrokerAll(true, true, this.brokerController.getBrokerConfig().isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        LOGGER.info("Finish to change to master brokerName={}", this.brokerController.getBrokerConfig().getBrokerName());
+    }
+
     @Override public void startup() {
 
     }

broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java

@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.failover;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.MessageQueueSelector;
+import org.apache.rocketmq.client.producer.SendCallback;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+
+public class EscapeBridge {
+    protected static final InternalLogger LOG = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final String innerProducerGroupName;
+    private final String innerConsumerGroupName;
+
+    private final BrokerController brokerController;
+
+    private DefaultMQProducer innerProducer;
+    private DefaultMQPullConsumer innerConsumer;
+
+    public EscapeBridge(BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.innerProducerGroupName = "InnerProducerGroup_" + brokerController.getBrokerConfig().getBrokerName() + "_" + brokerController.getBrokerConfig().getBrokerId();
+        this.innerConsumerGroupName = "InnerConsumerGroup_" + brokerController.getBrokerConfig().getBrokerName() + "_" + brokerController.getBrokerConfig().getBrokerId();
+    }
+
+    public void start() throws Exception {
+        if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) {
+            String nameserver = this.brokerController.getNameServerList();
+            if (nameserver != null && !nameserver.isEmpty()) {
+                startInnerProducer(nameserver);
+                startInnerConsumer(nameserver);
+                LOG.info("start inner producer and consumer success.");
+            } else {
+                throw new RuntimeException("nameserver address is null or empty");
+            }
+        }
+    }
+
+    public void shutdown() {
+        if (this.innerProducer != null) {
+            this.innerProducer.shutdown();
+        }
+
+        if (this.innerConsumer != null) {
+            this.innerConsumer.shutdown();
+        }
+    }
+
+    private void startInnerProducer(String nameServer) throws MQClientException {
+        try {
+            innerProducer = new DefaultMQProducer(innerProducerGroupName);
+            innerProducer.setNamesrvAddr(nameServer);
+            innerProducer.start();
+        } catch (MQClientException e) {
+            LOG.error("start inner producer failed, nameserver address: {}", nameServer, e);
+            throw e;
+        }
+    }
+
+    private void startInnerConsumer(String nameServer) throws MQClientException {
+        try {
+            innerConsumer = new DefaultMQPullConsumer(innerConsumerGroupName);
+            innerConsumer.setNamesrvAddr(nameServer);
+            innerConsumer.start();
+        } catch (MQClientException e) {
+            LOG.error("start inner consumer failed, nameserver address: {}", nameServer, e);
+            throw e;
+        }
+    }
+
+    public PutMessageResult putMessage(MessageExtBrokerInner messageExt) {
+        BrokerController masterBroker = this.brokerController.peekMasterBroker();
+        if (masterBroker != null) {
+            return masterBroker.getMessageStore().putMessage(messageExt);
+        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
+            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()
+            && innerProducer != null) {
+            // Remote Acting lead to born timestamp, msgId changed, it need to polish.
+            try {
+                messageExt.setWaitStoreMsgOK(false);
+                SendResult sendResult = innerProducer.send(messageExt);
+                return transformSendResult2PutResult(sendResult);
+            } catch (Exception e) {
+                LOG.error("sendMessageInFailover to remote failed", e);
+                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+            }
+        } else {
+            LOG.warn("Put message failed, enableSlaveActingMaster={}, enableRemoteEscape={}.",
+                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());
+            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+        }
+    }
+
+    public CompletableFuture<PutMessageResult> asyncPutMessage(MessageExtBrokerInner messageExt) {
+        BrokerController masterBroker = this.brokerController.peekMasterBroker();
+        CompletableFuture<PutMessageResult> completableFuture = new CompletableFuture<>();
+        if (masterBroker != null) {
+            return masterBroker.getMessageStore().asyncPutMessage(messageExt);
+        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
+            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()
+            && innerProducer != null) {
+            // Remote Acting lead to born timestamp, msgId changed, it need to polish.
+            try {
+                messageExt.setWaitStoreMsgOK(false);
+                innerProducer.send(messageExt, new SendCallback() {
+                    @Override public void onSuccess(SendResult sendResult) {
+                        completableFuture.complete(transformSendResult2PutResult(sendResult));
+                    }
+
+                    @Override public void onException(Throwable e) {
+                        completableFuture.complete(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
+                    }
+                });
+                return completableFuture;
+            } catch (Exception e) {
+                LOG.error("sendMessageInFailover to remote failed", e);
+                return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true));
+            }
+        } else {
+            LOG.warn("Put message failed, enableSlaveActingMaster={}, enableRemoteEscape={}.",
+                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());
+            return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null));
+        }
+    }
+
+    public PutMessageResult putMessageToSpecificQueue(MessageExtBrokerInner messageExt) {
+        BrokerController masterBroker = this.brokerController.peekMasterBroker();
+        if (masterBroker != null) {
+            return masterBroker.getMessageStore().putMessage(messageExt);
+        } else if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()
+            && this.brokerController.getBrokerConfig().isEnableRemoteEscape()
+            && this.innerProducer != null) {
+            try {
+                messageExt.setWaitStoreMsgOK(false);
+                // Remote Acting lead to born timestamp, msgId changed, it need to polish.
+                SendResult sendResult = innerProducer.send(messageExt, new MessageQueueSelector() {
+                    @Override
+                    public MessageQueue select(List<MessageQueue> mqs, Message msg, Object arg) {
+                        String id = (String) arg;
+                        int index = Math.abs(id.hashCode()) % mqs.size();
+                        return mqs.get(index);
+                    }
+                }, messageExt.getTopic() + messageExt.getStoreHost());
+                return transformSendResult2PutResult(sendResult);
+            } catch (Exception e) {
+                LOG.error("sendMessageInFailover to remote failed", e);
+                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+            }
+        } else {
+            LOG.warn("Put message to specific queue failed, enableSlaveActingMaster={}, enableRemoteEscape={}.",
+                this.brokerController.getBrokerConfig().isEnableSlaveActingMaster(), this.brokerController.getBrokerConfig().isEnableRemoteEscape());
+            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+        }
+    }
+
+    private PutMessageResult transformSendResult2PutResult(SendResult sendResult) {
+        if (sendResult == null) {
+            return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+        }
+        switch (sendResult.getSendStatus()) {
+            case SEND_OK:
+                return new PutMessageResult(PutMessageStatus.PUT_OK, null, true);
+            case SLAVE_NOT_AVAILABLE:
+                return new PutMessageResult(PutMessageStatus.SLAVE_NOT_AVAILABLE, null, true);
+            case FLUSH_DISK_TIMEOUT:
+                return new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, null, true);
+            case FLUSH_SLAVE_TIMEOUT:
+                return new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, null, true);
+            default:
+                return new PutMessageResult(PutMessageStatus.PUT_TO_REMOTE_BROKER_FAIL, null, true);
+        }
+    }
+
+    public MessageExt getMessage(String topic, long offset, int queueId, String brokerName) {
+        MessageStore messageStore = brokerController.getMessageStoreByBrokerName(brokerName);
+        if (messageStore != null) {
+            final GetMessageResult getMessageTmpResult = messageStore.getMessage(innerConsumerGroupName, topic, queueId, offset, 1, null);
+            List<MessageExt> list = decodeMsgList(getMessageTmpResult);
+            if (list == null || list.isEmpty()) {
+                LOG.warn("Can not get msg , topic {}, offset {}, queueId {}, result is {}", topic, offset, queueId, getMessageTmpResult);
+                return null;
+            } else {
+                return list.get(0);
+            }
+        } else if (innerConsumer != null) {
+            return getMessageFromRemote(topic, offset, queueId, brokerName);
+        } else {
+            return null;
+        }
+    }
+
+    protected List<MessageExt> decodeMsgList(GetMessageResult getMessageResult) {
+        List<MessageExt> foundList = new ArrayList<>();
+        try {
+            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
+            if (messageBufferList != null) {
+                for (int i = 0; i < messageBufferList.size(); i++) {
+                    ByteBuffer bb = messageBufferList.get(i);
+                    if (bb == null) {
+                        LOG.error("bb is null {}", getMessageResult);
+                        continue;
+                    }
+                    MessageExt msgExt = MessageDecoder.decode(bb);
+                    if (msgExt == null) {
+                        LOG.error("decode msgExt is null {}", getMessageResult);
+                        continue;
+                    }
+                    // use CQ offset, not offset in Message
+                    msgExt.setQueueOffset(getMessageResult.getMessageQueueOffset().get(i));
+                    foundList.add(msgExt);
+                }
+            }
+        } finally {
+            getMessageResult.release();
+        }
+
+        return foundList;
+    }
+
+    protected MessageExt getMessageFromRemote(String topic, long offset, int queueId, String brokerName) {
+        try {
+            PullResult pullResult = innerConsumer.pull(new MessageQueue(topic, brokerName, queueId), "*", offset, 1);
+            if (pullResult.getPullStatus().equals(PullStatus.FOUND)) {
+                return pullResult.getMsgFoundList().get(0);
+            }
+        } catch (Exception e) {
+            LOG.error("Get message from remote failed.", e);
+        }
+
+        return null;
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/filtersrv/FilterServerManager.java

@@ -29,6 +29,7 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerStartup;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
@@ -52,9 +53,8 @@ public class FilterServerManager {
 
     public void start() {
 
-        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-            @Override
-            public void run() {
+        this.scheduledExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(brokerController.getBrokerConfig()) {
+            @Override public void run2() {
                 try {
                     FilterServerManager.this.createFilterServer();
                 } catch (Exception e) {

broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java

@@ -17,25 +17,35 @@
 package org.apache.rocketmq.broker.latency;
 
 import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.netty.RequestTask;
 import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode;
 
+/**
+ * BrokerFastFailure will cover {@link BrokerController#getSendThreadPoolQueue()} and {@link
+ * BrokerController#getPullThreadPoolQueue()}
+ */
 public class BrokerFastFailure {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-    private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl(
-        "BrokerFastFailureScheduledThread"));
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final ScheduledExecutorService scheduledExecutorService;
     private final BrokerController brokerController;
 
+    private volatile long jstackTime = System.currentTimeMillis();
+
     public BrokerFastFailure(final BrokerController brokerController) {
         this.brokerController = brokerController;
+        this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1,
+            new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true,
+                brokerController == null ? null : brokerController.getBrokerConfig()));
     }
 
     public static RequestTask castRunnable(final Runnable runnable) {
@@ -45,16 +55,15 @@ public class BrokerFastFailure {
                 return (RequestTask) object.getRunnable();
             }
         } catch (Throwable e) {
-            log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e);
+            LOGGER.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e);
         }
 
         return null;
     }
 
     public void start() {
-        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-            @Override
-            public void run() {
+        this.scheduledExecutorService.scheduleAtFixedRate(new AbstractBrokerRunnable(this.brokerController.getBrokerConfig()) {
+            @Override public void run2() {
                 if (brokerController.getBrokerConfig().isBrokerFastFailureEnable()) {
                     cleanExpiredRequest();
                 }
@@ -63,6 +72,7 @@ public class BrokerFastFailure {
     }
 
     private void cleanExpiredRequest() {
+
         while (this.brokerController.getMessageStore().isOSPageCacheBusy()) {
             try {
                 if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) {
@@ -72,7 +82,13 @@ public class BrokerFastFailure {
                     }
 
                     final RequestTask rt = castRunnable(runnable);
-                    rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", System.currentTimeMillis() - rt.getCreateTimestamp(), this.brokerController.getSendThreadPoolQueue().size()));
+                    if (rt != null) {
+                        rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format(
+                            "[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, "
+                                + "size of queue: %d",
+                            System.currentTimeMillis() - rt.getCreateTimestamp(),
+                            this.brokerController.getSendThreadPoolQueue().size()));
+                    }
                 } else {
                     break;
                 }
@@ -86,11 +102,17 @@ public class BrokerFastFailure {
         cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(),
             this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue());
 
+        cleanExpiredRequestInQueue(this.brokerController.getLitePullThreadPoolQueue(),
+            this.brokerController.getBrokerConfig().getWaitTimeMillsInLitePullQueue());
+
         cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(),
             this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue());
 
         cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this
             .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue());
+
+        cleanExpiredRequestInQueue(this.brokerController.getAckThreadPoolQueue(),
+            brokerController.getBrokerConfig().getWaitTimeMillsInAckQueue());
     }
 
     void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) {
@@ -111,6 +133,10 @@ public class BrokerFastFailure {
                         if (blockingQueue.remove(runnable)) {
                             rt.setStopRun(true);
                             rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", behind, blockingQueue.size()));
+                            if (System.currentTimeMillis() - jstackTime > 15000) {
+                                jstackTime = System.currentTimeMillis();
+                                LOGGER.warn("broker jstack \n " + UtilAll.jstack());
+                            }
                         }
                     } else {
                         break;

broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java

@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.loadbalance;
+
+import com.google.common.collect.Lists;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.impl.factory.MQClientInstance;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+
+public class AssignmentManager {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    private transient BrokerController brokerController;
+
+    private final ConcurrentHashMap<String, Set<MessageQueue>> topicSubscribeInfoTable = new ConcurrentHashMap<String, Set<MessageQueue>>();
+
+    private ScheduledExecutorService scheduledExecutorService;
+
+    private static final List<String> IGNORE_ROUTE_TOPICS = Lists.newArrayList(
+        MixAll.CID_RMQ_SYS_PREFIX,
+        MixAll.DEFAULT_CONSUMER_GROUP,
+        MixAll.TOOLS_CONSUMER_GROUP,
+        MixAll.FILTERSRV_CONSUMER_GROUP,
+        MixAll.MONITOR_CONSUMER_GROUP,
+        MixAll.ONS_HTTP_PROXY_GROUP,
+        MixAll.CID_ONSAPI_PERMISSION_GROUP,
+        MixAll.CID_ONSAPI_OWNER_GROUP,
+        MixAll.CID_ONSAPI_PULL_GROUP
+    );
+
+    private final List<String> ignoreRouteTopics = Lists.newArrayList(IGNORE_ROUTE_TOPICS);
+
+    public AssignmentManager(BrokerController brokerController) {
+        this.brokerController = brokerController;
+        ignoreRouteTopics.add(brokerController.getBrokerConfig().getBrokerClusterName());
+        ignoreRouteTopics.add(brokerController.getBrokerConfig().getBrokerName());
+        scheduledExecutorService = Executors
+            .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("LoadBalanceManagerScheduledThread", brokerController.getBrokerIdentity()));
+    }
+
+    public void start() {
+        this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+
+            @Override
+            public void run() {
+                try {
+                    updateTopicRouteInfoFromNameServer();
+                } catch (Exception e) {
+                    log.error("ScheduledTask: failed to pull TopicRouteData from NameServer", e);
+                }
+            }
+        }, 1000, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS);
+    }
+
+    public void shutdown() {
+        this.scheduledExecutorService.shutdown();
+    }
+
+    public void updateTopicRouteInfoFromNameServer() {
+        Set<String> topicList = new HashSet<>(brokerController.getTopicConfigManager().getTopicConfigTable().keySet());
+
+        LOOP:
+        for (String topic : topicList) {
+            for (String keyword : ignoreRouteTopics) {
+                if (topic.contains(keyword) || TopicValidator.isSystemTopic(topic)) {
+                    continue LOOP;
+                }
+            }
+
+            this.updateTopicRouteInfoFromNameServer(topic);
+        }
+    }
+
+    public boolean updateTopicRouteInfoFromNameServer(final String topic) {
+        try {
+            TopicRouteData topicRouteData = this.brokerController.getBrokerOuterAPI().getTopicRouteInfoFromNameServer(topic, 1000 * 3);
+            if (topicRouteData != null) {
+                Set<MessageQueue> newSubscribeInfo = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
+                Set<MessageQueue> oldSubscribeInfo = topicSubscribeInfoTable.get(topic);
+                boolean changed = !newSubscribeInfo.equals(oldSubscribeInfo);
+
+                if (changed) {
+                    log.info("the topic[{}] subscribe message queue changed, old[{}] ,new[{}]", topic, oldSubscribeInfo, newSubscribeInfo);
+                    topicSubscribeInfoTable.put(topic, newSubscribeInfo);
+                    return true;
+                }
+            } else {
+                log.warn("updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", topic);
+            }
+        } catch (Exception e) {
+            if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+                log.warn("updateTopicRouteInfoFromNameServer Exception", e);
+                if (e instanceof MQBrokerException && ResponseCode.TOPIC_NOT_EXIST == ((MQBrokerException) e).getResponseCode()) {
+                    // clean no used topic
+                    cleanNoneRouteTopic(topic);
+                }
+            }
+        }
+        return false;
+    }
+
+    private void cleanNoneRouteTopic(String topic) {
+        // clean no used topic
+        topicSubscribeInfoTable.remove(topic);
+    }
+
+    public Set<MessageQueue> getTopicSubscribeInfo(String topic) {
+        return topicSubscribeInfoTable.get(topic);
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/loadbalance/MessageRequestModeManager.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.loadbalance;
+
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.BrokerPathConfigHelper;
+import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+
+public class MessageRequestModeManager extends ConfigManager {
+
+    private transient BrokerController brokerController;
+
+    private ConcurrentHashMap<String/*topic*/, ConcurrentHashMap<String/*consumerGroup*/, SetMessageRequestModeRequestBody>>
+        messageRequestModeMap = new ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>>();
+
+    public MessageRequestModeManager() {
+        // empty construct for decode
+    }
+
+    public MessageRequestModeManager(BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    public void setMessageRequestMode(String topic, String consumerGroup, SetMessageRequestModeRequestBody requestBody) {
+        ConcurrentHashMap<String, SetMessageRequestModeRequestBody> consumerGroup2ModeMap = messageRequestModeMap.get(topic);
+        if (consumerGroup2ModeMap == null) {
+            consumerGroup2ModeMap = new ConcurrentHashMap<String, SetMessageRequestModeRequestBody>();
+            ConcurrentHashMap<String, SetMessageRequestModeRequestBody> pre =
+                messageRequestModeMap.putIfAbsent(topic, consumerGroup2ModeMap);
+            if (pre != null) {
+                consumerGroup2ModeMap = pre;
+            }
+        }
+        consumerGroup2ModeMap.put(consumerGroup, requestBody);
+    }
+
+    public SetMessageRequestModeRequestBody getMessageRequestMode(String topic, String consumerGroup) {
+        ConcurrentHashMap<String, SetMessageRequestModeRequestBody> consumerGroup2ModeMap = messageRequestModeMap.get(topic);
+        if (consumerGroup2ModeMap != null) {
+            return consumerGroup2ModeMap.get(consumerGroup);
+        }
+
+        return null;
+    }
+
+    public ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> getMessageRequestModeMap() {
+        return this.messageRequestModeMap;
+    }
+
+    public void setMessageRequestModeMap(ConcurrentHashMap<String, ConcurrentHashMap<String, SetMessageRequestModeRequestBody>> messageRequestModeMap) {
+        this.messageRequestModeMap = messageRequestModeMap;
+    }
+
+    @Override
+    public String encode() {
+        return this.encode(false);
+    }
+
+    @Override
+    public String configFilePath() {
+        return BrokerPathConfigHelper.getMessageRequestModePath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
+    }
+
+    @Override
+    public void decode(String jsonString) {
+        if (jsonString != null) {
+            MessageRequestModeManager obj = RemotingSerializable.fromJson(jsonString, MessageRequestModeManager.class);
+            if (obj != null) {
+                this.messageRequestModeMap = obj.messageRequestModeMap;
+            }
+        }
+    }
+
+    @Override
+    public String encode(boolean prettyFormat) {
+        return RemotingSerializable.toJson(this, prettyFormat);
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java

@@ -32,6 +32,9 @@ public class LmqPullRequestHoldService extends PullRequestHoldService {
 
     @Override
     public String getServiceName() {
+        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return this.brokerController.getBrokerIdentity().getLoggerIdentifier() + LmqPullRequestHoldService.class.getSimpleName();
+        }
         return LmqPullRequestHoldService.class.getSimpleName();
     }
 

broker/src/main/java/org/apache/rocketmq/broker/longpolling/ManyPullRequest.java

@@ -43,4 +43,8 @@ public class ManyPullRequest {
     public ArrayList<PullRequest> getPullRequestList() {
         return pullRequestList;
     }
+
+    public synchronized boolean isEmpty() {
+        return this.pullRequestList.isEmpty();
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotificationRequest.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.longpolling;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+import io.netty.channel.Channel;
+
+public class NotificationRequest {
+    private RemotingCommand remotingCommand;
+    private Channel channel;
+    private long expired;
+    private AtomicBoolean complete = new AtomicBoolean(false);
+
+    public NotificationRequest(RemotingCommand remotingCommand, Channel channel, long expired) {
+        this.channel = channel;
+        this.remotingCommand = remotingCommand;
+        this.expired = expired;
+    }
+
+    public Channel getChannel() {
+        return channel;
+    }
+
+    public RemotingCommand getRemotingCommand() {
+        return remotingCommand;
+    }
+
+    public boolean isTimeout() {
+        return System.currentTimeMillis() > (expired - 3000);
+    }
+
+    public boolean complete() {
+        return complete.compareAndSet(false, true);
+    }
+
+    @Override
+    public String toString() {
+        return remotingCommand.toString();
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/longpolling/NotifyMessageArrivingListener.java

@@ -17,21 +17,29 @@
 
 package org.apache.rocketmq.broker.longpolling;
 
+import org.apache.rocketmq.broker.processor.NotificationProcessor;
+import org.apache.rocketmq.broker.processor.PopMessageProcessor;
 import org.apache.rocketmq.store.MessageArrivingListener;
 
 import java.util.Map;
 
 public class NotifyMessageArrivingListener implements MessageArrivingListener {
     private final PullRequestHoldService pullRequestHoldService;
+    private final PopMessageProcessor popMessageProcessor;
+    private final NotificationProcessor notificationProcessor;
 
-    public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService) {
+    public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService, final PopMessageProcessor popMessageProcessor, final NotificationProcessor notificationProcessor) {
         this.pullRequestHoldService = pullRequestHoldService;
+        this.popMessageProcessor = popMessageProcessor;
+        this.notificationProcessor = notificationProcessor;
     }
 
     @Override
     public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
-        long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
+                         long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
         this.pullRequestHoldService.notifyMessageArriving(topic, queueId, logicOffset, tagsCode,
             msgStoreTime, filterBitMap, properties);
+        this.popMessageProcessor.notifyMessageArriving(topic, queueId);
+        this.notificationProcessor.notifyMessageArriving(topic, queueId);
     }
 }

broker/src/main/java/org/apache/rocketmq/broker/longpolling/PopRequest.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.longpolling;
+
+import java.util.Comparator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+import io.netty.channel.Channel;
+
+public class PopRequest {
+    private static final AtomicLong COUNTER = new AtomicLong(Long.MIN_VALUE);
+
+    private RemotingCommand remotingCommand;
+    private Channel channel;
+    private long expired;
+    private AtomicBoolean complete = new AtomicBoolean(false);
+    private final long op = COUNTER.getAndIncrement();
+
+    public PopRequest(RemotingCommand remotingCommand, Channel channel, long expired) {
+        this.channel = channel;
+        this.remotingCommand = remotingCommand;
+        this.expired = expired;
+    }
+
+    public Channel getChannel() {
+        return channel;
+    }
+
+    public RemotingCommand getRemotingCommand() {
+        return remotingCommand;
+    }
+
+    public boolean isTimeout() {
+        return System.currentTimeMillis() > (expired - 50);
+    }
+
+    public boolean complete() {
+        return complete.compareAndSet(false, true);
+    }
+
+    public long getExpired() {
+        return expired;
+    }
+
+    @Override
+    public String toString() {
+        final StringBuilder sb = new StringBuilder("PopRequest{");
+        sb.append("cmd=").append(remotingCommand);
+        sb.append(", channel=").append(channel);
+        sb.append(", expired=").append(expired);
+        sb.append(", complete=").append(complete);
+        sb.append(", op=").append(op);
+        sb.append('}');
+        return sb.toString();
+    }
+
+    public static final Comparator<PopRequest> COMPARATOR = new Comparator<PopRequest>() {
+        @Override
+        public int compare(PopRequest o1, PopRequest o2) {
+            int ret = (int) (o1.getExpired() - o2.getExpired());
+
+            if (ret != 0) {
+                return ret;
+            }
+            ret = (int) (o1.op - o2.op);
+            if (ret != 0) {
+                return ret;
+            }
+            return -1;
+        }
+    };
+}

broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java

@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.ServiceThread;
 import org.apache.rocketmq.common.SystemClock;
@@ -78,7 +79,7 @@ public class PullRequestHoldService extends ServiceThread {
                 this.checkHoldRequest();
                 long costTime = this.systemClock.now() - beginLockTimestamp;
                 if (costTime > 5 * 1000) {
-                    log.info("[NOTIFYME] check hold request cost {} ms.", costTime);
+                    log.warn("PullRequestHoldService: check hold pull request cost {}ms", costTime);
                 }
             } catch (Throwable e) {
                 log.warn(this.getServiceName() + " service has exception. ", e);
@@ -90,6 +91,9 @@ public class PullRequestHoldService extends ServiceThread {
 
     @Override
     public String getServiceName() {
+        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return this.brokerController.getBrokerIdentity().getLoggerIdentifier() + PullRequestHoldService.class.getSimpleName();
+        }
         return PullRequestHoldService.class.getSimpleName();
     }
 
@@ -103,7 +107,9 @@ public class PullRequestHoldService extends ServiceThread {
                 try {
                     this.notifyMessageArriving(topic, queueId, offset);
                 } catch (Throwable e) {
-                    log.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
+                    log.error(
+                        "PullRequestHoldService: failed to check hold request failed, topic={}, queueId={}", topic,
+                        queueId, e);
                 }
             }
         }
@@ -141,7 +147,9 @@ public class PullRequestHoldService extends ServiceThread {
                                 this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
                                     request.getRequestCommand());
                             } catch (Throwable e) {
-                                log.error("execute request when wakeup failed.", e);
+                                log.error(
+                                    "PullRequestHoldService#notifyMessageArriving: failed to execute request when "
+                                        + "message matched, topic={}, queueId={}", topic, queueId, e);
                             }
                             continue;
                         }
@@ -152,7 +160,9 @@ public class PullRequestHoldService extends ServiceThread {
                             this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
                                 request.getRequestCommand());
                         } catch (Throwable e) {
-                            log.error("execute request when wakeup failed.", e);
+                            log.error(
+                                "PullRequestHoldService#notifyMessageArriving: failed to execute request when time's "
+                                    + "up, topic={}, queueId={}", topic, queueId, e);
                         }
                         continue;
                     }
@@ -166,4 +176,22 @@ public class PullRequestHoldService extends ServiceThread {
             }
         }
     }
+
+    public void notifyMasterOnline() {
+        for (ManyPullRequest mpr : this.pullRequestTable.values()) {
+            if (mpr == null || mpr.isEmpty()) {
+                continue;
+            }
+            for (PullRequest request : mpr.cloneListAndClear()) {
+                try {
+                    log.info("notify master online, wakeup {} {}", request.getClientChannel(), request.getRequestCommand());
+                    this.brokerController.getPullMessageProcessor().executeRequestWhenWakeup(request.getClientChannel(),
+                        request.getRequestCommand());
+                } catch (Throwable e) {
+                    log.error("execute request when master online failed.", e);
+                }
+            }
+        }
+
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/mqtrace/AbortProcessException.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.mqtrace;
+
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.help.FAQUrl;
+
+/**
+ *
+ * This exception is used for broker hooks only : SendMessageHook, ConsumeMessageHook, pullMessageHook
+ * This exception is not ignored while executing hooks and it means that
+ * certain processor should return an immediate error response to the client. The
+ * error response code is included in AbortProcessException.  it's naming might
+ * be confusing, so feel free to refactor this class. Also when any class implements
+ * the 3 hook interface mentioned above we should be careful if we want to throw
+ * an AbortProcessException, because it will change the control flow of broker
+ * and cause a RemotingCommand return error immediately. So be aware of the side
+ * effect before throw AbortProcessException in your implementation.
+ *
+ */
+public class AbortProcessException extends RuntimeException {
+    private static final long serialVersionUID = -5728810933841185841L;
+    private int responseCode;
+    private String errorMessage;
+
+    public AbortProcessException(String errorMessage, Throwable cause) {
+        super(FAQUrl.attachDefaultURL(errorMessage), cause);
+        this.responseCode = -1;
+        this.errorMessage = errorMessage;
+    }
+
+    public AbortProcessException(int responseCode, String errorMessage) {
+        super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + "  DESC: "
+            + errorMessage));
+        this.responseCode = responseCode;
+        this.errorMessage = errorMessage;
+    }
+
+    public int getResponseCode() {
+        return responseCode;
+    }
+
+    public AbortProcessException setResponseCode(final int responseCode) {
+        this.responseCode = responseCode;
+        return this;
+    }
+
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+
+    public void setErrorMessage(final String errorMessage) {
+        this.errorMessage = errorMessage;
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/mqtrace/ConsumeMessageContext.java

@@ -17,6 +17,8 @@
 package org.apache.rocketmq.broker.mqtrace;
 
 import java.util.Map;
+
+import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class ConsumeMessageContext {
@@ -30,13 +32,22 @@ public class ConsumeMessageContext {
     private boolean success;
     private String status;
     private Object mqTraceContext;
+    private TopicConfig topicConfig;
+
+    private String accountAuthType;
+    private String accountOwnerParent;
+    private String accountOwnerSelf;
+    private int rcvMsgNum;
+    private int rcvMsgSize;
+    private BrokerStatsManager.StatsType rcvStat;
+    private int commercialRcvMsgNum;
 
     private String commercialOwner;
     private BrokerStatsManager.StatsType commercialRcvStats;
     private int commercialRcvTimes;
     private int commercialRcvSize;
-    private String namespace;
 
+    private String namespace;
     public String getConsumerGroup() {
         return consumerGroup;
     }
@@ -109,6 +120,14 @@ public class ConsumeMessageContext {
         this.mqTraceContext = mqTraceContext;
     }
 
+    public TopicConfig getTopicConfig() {
+        return topicConfig;
+    }
+
+    public void setTopicConfig(TopicConfig topicConfig) {
+        this.topicConfig = topicConfig;
+    }
+
     public int getBodyLength() {
         return bodyLength;
     }
@@ -117,6 +136,62 @@ public class ConsumeMessageContext {
         this.bodyLength = bodyLength;
     }
 
+    public String getAccountAuthType() {
+        return accountAuthType;
+    }
+
+    public void setAccountAuthType(String accountAuthType) {
+        this.accountAuthType = accountAuthType;
+    }
+
+    public String getAccountOwnerParent() {
+        return accountOwnerParent;
+    }
+
+    public void setAccountOwnerParent(String accountOwnerParent) {
+        this.accountOwnerParent = accountOwnerParent;
+    }
+
+    public String getAccountOwnerSelf() {
+        return accountOwnerSelf;
+    }
+
+    public void setAccountOwnerSelf(String accountOwnerSelf) {
+        this.accountOwnerSelf = accountOwnerSelf;
+    }
+
+    public int getRcvMsgNum() {
+        return rcvMsgNum;
+    }
+
+    public void setRcvMsgNum(int rcvMsgNum) {
+        this.rcvMsgNum = rcvMsgNum;
+    }
+
+    public int getRcvMsgSize() {
+        return rcvMsgSize;
+    }
+
+    public void setRcvMsgSize(int rcvMsgSize) {
+        this.rcvMsgSize = rcvMsgSize;
+    }
+
+    public BrokerStatsManager.StatsType getRcvStat() {
+        return rcvStat;
+    }
+
+    public void setRcvStat(BrokerStatsManager.StatsType rcvStat) {
+        this.rcvStat = rcvStat;
+    }
+
+    public int getCommercialRcvMsgNum() {
+        return commercialRcvMsgNum;
+    }
+
+    public void setCommercialRcvMsgNum(int commercialRcvMsgNum) {
+        this.commercialRcvMsgNum = commercialRcvMsgNum;
+    }
+
     public String getCommercialOwner() {
         return commercialOwner;
     }

broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageContext.java

@@ -17,11 +17,16 @@
 package org.apache.rocketmq.broker.mqtrace;
 
 import java.util.Properties;
+
 import org.apache.rocketmq.common.message.MessageType;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class SendMessageContext {
+    /** namespace */
+    private String namespace;
+    /** producer group without namespace. */
     private String producerGroup;
+    /** topic without namespace. */
     private String topic;
     private String msgId;
     private String originMsgId;
@@ -38,14 +43,37 @@ public class SendMessageContext {
     private String brokerRegionId;
     private String msgUniqueKey;
     private long bornTimeStamp;
+    private long requestTimeStamp;
     private MessageType msgType = MessageType.Trans_msg_Commit;
+
     private boolean isSuccess = false;
 
+    /**
+     * Account Statistics
+     */
+    private String accountAuthType;
+    private String accountOwnerParent;
+    private String accountOwnerSelf;
+    private int sendMsgNum;
+    private int sendMsgSize;
+    private BrokerStatsManager.StatsType sendStat;
+    private int commercialSendMsgNum;
+
+    /**
+     * For Commercial
+     */
     private String commercialOwner;
     private BrokerStatsManager.StatsType commercialSendStats;
     private int commercialSendSize;
     private int commercialSendTimes;
-    private String namespace;
+
+    public String getNamespace() {
+        return namespace;
+    }
+
+    public void setNamespace(String namespace) {
+        this.namespace = namespace;
+    }
 
     public boolean isSuccess() {
         return isSuccess;
@@ -79,6 +107,14 @@ public class SendMessageContext {
         this.bornTimeStamp = bornTimeStamp;
     }
 
+    public long getRequestTimeStamp() {
+        return requestTimeStamp;
+    }
+
+    public void setRequestTimeStamp(long requestTimeStamp) {
+        this.requestTimeStamp = requestTimeStamp;
+    }
+
     public String getBrokerRegionId() {
         return brokerRegionId;
     }
@@ -207,10 +243,66 @@ public class SendMessageContext {
         this.commercialOwner = commercialOwner;
     }
 
+    public String getAccountAuthType() {
+        return accountAuthType;
+    }
+
+    public void setAccountAuthType(String accountAuthType) {
+        this.accountAuthType = accountAuthType;
+    }
+
+    public String getAccountOwnerParent() {
+        return accountOwnerParent;
+    }
+
+    public void setAccountOwnerParent(String accountOwnerParent) {
+        this.accountOwnerParent = accountOwnerParent;
+    }
+
+    public String getAccountOwnerSelf() {
+        return accountOwnerSelf;
+    }
+
+    public void setAccountOwnerSelf(String accountOwnerSelf) {
+        this.accountOwnerSelf = accountOwnerSelf;
+    }
+
+    public int getSendMsgNum() {
+        return sendMsgNum;
+    }
+
+    public void setSendMsgNum(int sendMsgNum) {
+        this.sendMsgNum = sendMsgNum;
+    }
+
+    public int getSendMsgSize() {
+        return sendMsgSize;
+    }
+
+    public void setSendMsgSize(int sendMsgSize) {
+        this.sendMsgSize = sendMsgSize;
+    }
+
+    public BrokerStatsManager.StatsType getSendStat() {
+        return sendStat;
+    }
+
+    public void setSendStat(BrokerStatsManager.StatsType sendStat) {
+        this.sendStat = sendStat;
+    }
+
     public BrokerStatsManager.StatsType getCommercialSendStats() {
         return commercialSendStats;
     }
 
+    public int getCommercialSendMsgNum() {
+        return commercialSendMsgNum;
+    }
+
+    public void setCommercialSendMsgNum(int commercialSendMsgNum) {
+        this.commercialSendMsgNum = commercialSendMsgNum;
+    }
+
     public void setCommercialSendStats(final BrokerStatsManager.StatsType commercialSendStats) {
         this.commercialSendStats = commercialSendStats;
     }
@@ -230,12 +322,4 @@ public class SendMessageContext {
     public void setCommercialSendTimes(final int commercialSendTimes) {
         this.commercialSendTimes = commercialSendTimes;
     }
-
-    public String getNamespace() {
-        return namespace;
-    }
-
-    public void setNamespace(String namespace) {
-        this.namespace = namespace;
-    }
 }

broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManager.java

@@ -24,9 +24,11 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerPathConfigHelper;
 import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.DataVersion;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
@@ -34,14 +36,18 @@ import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 
 public class ConsumerOffsetManager extends ConfigManager {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-    protected static final String TOPIC_GROUP_SEPARATOR = "@";
+    private static final InternalLogger LOG = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    public static final String TOPIC_GROUP_SEPARATOR = "@";
+
+    private DataVersion dataVersion = new DataVersion();
 
     protected ConcurrentMap<String/* topic@group */, ConcurrentMap<Integer, Long>> offsetTable =
         new ConcurrentHashMap<String, ConcurrentMap<Integer, Long>>(512);
 
     protected transient BrokerController brokerController;
 
+    private transient AtomicLong versionChangeCounter = new AtomicLong(0);
+
     public ConsumerOffsetManager() {
     }
 
@@ -49,6 +55,36 @@ public class ConsumerOffsetManager extends ConfigManager {
         this.brokerController = brokerController;
     }
 
+    public void cleanOffset(String group) {
+        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
+        while (it.hasNext()) {
+            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
+            String topicAtGroup = next.getKey();
+            if (topicAtGroup.contains(group)) {
+                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+                if (arrays.length == 2 && group.equals(arrays[1])) {
+                    it.remove();
+                    LOG.warn("Clean group's offset, {}, {}", topicAtGroup, next.getValue());
+                }
+            }
+        }
+    }
+
+    public void cleanOffsetByTopic(String topic) {
+        Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
+        while (it.hasNext()) {
+            Entry<String, ConcurrentMap<Integer, Long>> next = it.next();
+            String topicAtGroup = next.getKey();
+            if (topicAtGroup.contains(topic)) {
+                String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+                if (arrays.length == 2 && topic.equals(arrays[0])) {
+                    it.remove();
+                    LOG.warn("Clean topic's offset, {}, {}", topicAtGroup, next.getValue());
+                }
+            }
+        }
+    }
+
     public void scanUnsubscribedTopic() {
         Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
         while (it.hasNext()) {
@@ -62,7 +98,7 @@ public class ConsumerOffsetManager extends ConfigManager {
                 if (null == brokerController.getConsumerManager().findSubscriptionData(group, topic)
                     && this.offsetBehindMuchThanData(topic, next.getValue())) {
                     it.remove();
-                    log.warn("remove topic offset, {}", topicAtGroup);
+                    LOG.warn("remove topic offset, {}", topicAtGroup);
                 }
             }
         }
@@ -118,6 +154,28 @@ public class ConsumerOffsetManager extends ConfigManager {
         return groups;
     }
 
+    public Map<String, Set<String>> getGroupTopicMap() {
+        Map<String, Set<String>> retMap = new HashMap<String, Set<String>>(128);
+
+        for (String key : this.offsetTable.keySet()) {
+            String[] arr = key.split(TOPIC_GROUP_SEPARATOR);
+            if (arr.length == 2) {
+                String topic = arr[0];
+                String group = arr[1];
+
+                Set<String> topics = retMap.get(group);
+                if (topics == null) {
+                    topics = new HashSet<String>(8);
+                    retMap.put(group, topics);
+                }
+
+                topics.add(topic);
+            }
+        }
+
+        return retMap;
+    }
+
     public void commitOffset(final String clientHost, final String group, final String topic, final int queueId,
         final long offset) {
         // topic@group
@@ -134,9 +192,13 @@ public class ConsumerOffsetManager extends ConfigManager {
         } else {
             Long storeOffset = map.put(queueId, offset);
             if (storeOffset != null && offset < storeOffset) {
-                log.warn("[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}", clientHost, key, queueId, offset, storeOffset);
+                LOG.warn("[NOTIFYME]update consumer offset less than store. clientHost={}, key={}, queueId={}, requestOffset={}, storeOffset={}", clientHost, key, queueId, offset, storeOffset);
             }
         }
+        if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getConsumerOffsetUpdateVersionStep() == 0) {
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
+        }
     }
 
     public long queryOffset(final String group, final String topic, final int queueId) {
@@ -145,8 +207,9 @@ public class ConsumerOffsetManager extends ConfigManager {
         ConcurrentMap<Integer, Long> map = this.offsetTable.get(key);
         if (null != map) {
             Long offset = map.get(queueId);
-            if (offset != null)
+            if (offset != null) {
                 return offset;
+            }
         }
 
         return -1;
@@ -167,10 +230,12 @@ public class ConsumerOffsetManager extends ConfigManager {
             ConsumerOffsetManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class);
             if (obj != null) {
                 this.offsetTable = obj.offsetTable;
+                this.dataVersion = obj.dataVersion;
             }
         }
     }
 
+    @Override
     public String encode(final boolean prettyFormat) {
         return RemotingSerializable.toJson(this, prettyFormat);
     }
@@ -232,6 +297,14 @@ public class ConsumerOffsetManager extends ConfigManager {
         }
     }
 
+    public DataVersion getDataVersion() {
+        return dataVersion;
+    }
+
+    public void setDataVersion(DataVersion dataVersion) {
+        this.dataVersion = dataVersion;
+    }
+
     public void removeOffset(final String group) {
         Iterator<Entry<String, ConcurrentMap<Integer, Long>>> it = this.offsetTable.entrySet().iterator();
         while (it.hasNext()) {
@@ -241,7 +314,7 @@ public class ConsumerOffsetManager extends ConfigManager {
                 String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
                 if (arrays.length == 2 && group.equals(arrays[1])) {
                     it.remove();
-                    log.warn("clean group offset {}", topicAtGroup);
+                    LOG.warn("clean group offset {}", topicAtGroup);
                 }
             }
         }

broker/src/main/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManager.java

@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.offset;
+
+import com.alibaba.fastjson.annotation.JSONField;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.BrokerPathConfigHelper;
+import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+
+public class ConsumerOrderInfoManager extends ConfigManager {
+
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private static final String TOPIC_GROUP_SEPARATOR = "@";
+    private static final long CLEAN_SPAN_FROM_LAST = 24 * 3600 * 1000;
+
+    private ConcurrentHashMap<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>> table =
+        new ConcurrentHashMap<>(128);
+
+    private transient BrokerController brokerController;
+
+    public ConsumerOrderInfoManager() {
+    }
+
+    public ConsumerOrderInfoManager(BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    public ConcurrentHashMap<String, ConcurrentHashMap<Integer, OrderInfo>> getTable() {
+        return table;
+    }
+
+    public void setTable(ConcurrentHashMap<String, ConcurrentHashMap<Integer, OrderInfo>> table) {
+        this.table = table;
+    }
+
+    /**
+     * not thread safe.
+     *
+     * @param topic
+     * @param group
+     * @param queueId
+     * @param msgOffsetList
+     */
+    public int update(String topic, String group, int queueId, List<Long> msgOffsetList) {
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);
+        if (qs == null) {
+            qs = new ConcurrentHashMap<>(16);
+            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> old = table.putIfAbsent(key, qs);
+            if (old != null) {
+                qs = old;
+            }
+        }
+
+        OrderInfo orderInfo = qs.get(queueId);
+
+        // start is same.
+        List<Long> simple = OrderInfo.simpleO(msgOffsetList);
+        if (orderInfo != null && simple.get(0).equals(orderInfo.getOffsetList().get(0))) {
+            if (simple.equals(orderInfo.getOffsetList())) {
+                orderInfo.setConsumedCount(orderInfo.getConsumedCount() + 1);
+            } else {
+                // reset, because msgs are changed.
+                orderInfo.setConsumedCount(0);
+            }
+            orderInfo.setLastConsumeTimestamp(System.currentTimeMillis());
+            orderInfo.setOffsetList(simple);
+            orderInfo.setCommitOffsetBit(0);
+        } else {
+            orderInfo = new OrderInfo();
+            orderInfo.setOffsetList(simple);
+            orderInfo.setLastConsumeTimestamp(System.currentTimeMillis());
+            orderInfo.setConsumedCount(0);
+            orderInfo.setCommitOffsetBit(0);
+
+            qs.put(queueId, orderInfo);
+        }
+
+        return orderInfo.getConsumedCount();
+    }
+
+    public boolean checkBlock(String topic, String group, int queueId, long invisibleTime) {
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);
+        if (qs == null) {
+            qs = new ConcurrentHashMap<>(16);
+            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> old = table.putIfAbsent(key, qs);
+            if (old != null) {
+                qs = old;
+            }
+        }
+
+        OrderInfo orderInfo = qs.get(queueId);
+
+        if (orderInfo == null) {
+            return false;
+        }
+
+        boolean isBlock = System.currentTimeMillis() - orderInfo.getLastConsumeTimestamp() < invisibleTime;
+
+        return isBlock && !orderInfo.isDone();
+    }
+
+    /**
+     * @param topic
+     * @param group
+     * @param queueId
+     * @param offset
+     * @return -1 : illegal, -2 : no need commit, >= 0 : commit
+     */
+    public long commitAndNext(String topic, String group, int queueId, long offset) {
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);
+
+        if (qs == null) {
+            return offset + 1;
+        }
+        OrderInfo orderInfo = qs.get(queueId);
+        if (orderInfo == null) {
+            log.warn("OrderInfo is null, {}, {}, {}", key, offset, orderInfo);
+            return offset + 1;
+        }
+
+        List<Long> offsetList = orderInfo.getOffsetList();
+        if (offsetList == null || offsetList.isEmpty()) {
+            log.warn("OrderInfo is empty, {}, {}, {}", key, offset, orderInfo);
+            return -1;
+        }
+        Long first = offsetList.get(0);
+        int i = 0, size = offsetList.size();
+        for (; i < size; i++) {
+            long temp;
+            if (i == 0) {
+                temp = first;
+            } else {
+                temp = first + offsetList.get(i);
+            }
+            if (offset == temp) {
+                break;
+            }
+        }
+        // not found
+        if (i >= size) {
+            log.warn("OrderInfo not found commit offset, {}, {}, {}", key, offset, orderInfo);
+            return -1;
+        }
+        //set bit
+        orderInfo.setCommitOffsetBit(orderInfo.getCommitOffsetBit() | (1L << i));
+        if (orderInfo.isDone()) {
+            if (size == 1) {
+                return offsetList.get(0) + 1;
+            } else {
+                return offsetList.get(size - 1) + first + 1;
+            }
+        }
+        return -2;
+    }
+
+    public OrderInfo get(String topic, String group, int queueId) {
+        String key = topic + TOPIC_GROUP_SEPARATOR + group;
+        ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = table.get(key);
+
+        if (qs == null) {
+            return null;
+        }
+
+        return qs.get(queueId);
+    }
+
+    public int getConsumeCount(String topic, String group, int queueId) {
+        OrderInfo orderInfo = get(topic, group, queueId);
+        return orderInfo == null ? 0 : orderInfo.getConsumedCount();
+    }
+
+    private void autoClean() {
+        if (brokerController == null) {
+            return;
+        }
+        Iterator<Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>>> iterator =
+            this.table.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>> entry =
+                iterator.next();
+            String topicAtGroup = entry.getKey();
+            ConcurrentHashMap<Integer/*queueId*/, OrderInfo> qs = entry.getValue();
+            String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR);
+            if (arrays.length != 2) {
+                continue;
+            }
+            String topic = arrays[0];
+            String group = arrays[1];
+
+            TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
+            if (topicConfig == null) {
+                iterator.remove();
+                log.info("Topic not exist, Clean order info, {}:{}", topicAtGroup, qs);
+                continue;
+            }
+
+            if (this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group) == null) {
+                iterator.remove();
+                log.info("Group not exist, Clean order info, {}:{}", topicAtGroup, qs);
+                continue;
+            }
+
+            if (qs.isEmpty()) {
+                iterator.remove();
+                log.info("Order table is empty, Clean order info, {}:{}", topicAtGroup, qs);
+                continue;
+            }
+
+            Iterator<Map.Entry<Integer/*queueId*/, OrderInfo>> qsIterator = qs.entrySet().iterator();
+            while (qsIterator.hasNext()) {
+                Map.Entry<Integer/*queueId*/, OrderInfo> qsEntry = qsIterator.next();
+
+                if (qsEntry.getKey() >= topicConfig.getReadQueueNums()) {
+                    qsIterator.remove();
+                    log.info("Queue not exist, Clean order info, {}:{}, {}", topicAtGroup, entry.getValue(), topicConfig);
+                    continue;
+                }
+
+                if (System.currentTimeMillis() - qsEntry.getValue().getLastConsumeTimestamp() > CLEAN_SPAN_FROM_LAST) {
+                    qsIterator.remove();
+                    log.info("Not consume long time, Clean order info, {}:{}, {}", topicAtGroup, entry.getValue(), topicConfig);
+                    continue;
+                }
+            }
+        }
+    }
+
+    @Override
+    public String encode() {
+        return this.encode(false);
+    }
+
+    @Override
+    public String configFilePath() {
+        if (brokerController != null) {
+            return BrokerPathConfigHelper.getConsumerOrderInfoPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
+        } else {
+            return BrokerPathConfigHelper.getConsumerOrderInfoPath("~");
+        }
+    }
+
+    @Override
+    public void decode(String jsonString) {
+        if (jsonString != null) {
+            ConsumerOrderInfoManager obj = RemotingSerializable.fromJson(jsonString, ConsumerOrderInfoManager.class);
+            if (obj != null) {
+                this.table = obj.table;
+            }
+        }
+    }
+
+    @Override
+    public String encode(boolean prettyFormat) {
+        this.autoClean();
+
+        StringBuilder stringBuilder = new StringBuilder();
+        stringBuilder.append("{\n").append("\t\"table\":{");
+        Iterator<Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>>> iterator =
+            this.table.entrySet().iterator();
+        int count1 = 0;
+        while (iterator.hasNext()) {
+            Map.Entry<String/* topic@group*/, ConcurrentHashMap<Integer/*queueId*/, OrderInfo>> entry =
+                iterator.next();
+            if (count1 > 0) {
+                stringBuilder.append(",");
+            }
+            stringBuilder.append("\n\t\t\"").append(entry.getKey()).append("\":{");
+            Iterator<Map.Entry<Integer/*queueId*/, OrderInfo>> qsIterator = entry.getValue().entrySet().iterator();
+            int count2 = 0;
+            while (qsIterator.hasNext()) {
+                Map.Entry<Integer/*queueId*/, OrderInfo> qsEntry = qsIterator.next();
+                if (count2 > 0) {
+                    stringBuilder.append(",");
+                }
+                stringBuilder.append("\n\t\t\t").append(qsEntry.getKey()).append(":")
+                    .append(qsEntry.getValue().encode());
+                count2++;
+            }
+            stringBuilder.append("\n\t\t}");
+            count1++;
+        }
+        stringBuilder.append("\n\t}").append("\n}");
+        return stringBuilder.toString();
+    }
+
+    public static class OrderInfo {
+        /**
+         * offset
+         */
+        private List<Long> offsetList;
+        /**
+         * consumed count
+         */
+        private int consumedCount;
+        /**
+         * last consume timestamp
+         */
+        private long lastConsumeTimestamp;
+        /**
+         * commit offset bit
+         */
+        private long commitOffsetBit;
+
+        public OrderInfo() {
+        }
+
+        public List<Long> getOffsetList() {
+            return offsetList;
+        }
+
+        public void setOffsetList(List<Long> offsetList) {
+            this.offsetList = offsetList;
+        }
+
+        public static List<Long> simpleO(List<Long> offsetList) {
+            List<Long> simple = new ArrayList<>();
+            if (offsetList.size() == 1) {
+                simple.addAll(offsetList);
+                return simple;
+            }
+            Long first = offsetList.get(0);
+            simple.add(first);
+            for (int i = 1; i < offsetList.size(); i++) {
+                simple.add(offsetList.get(i) - first);
+            }
+            return simple;
+        }
+
+        public int getConsumedCount() {
+            return consumedCount;
+        }
+
+        public void setConsumedCount(int consumedCount) {
+            this.consumedCount = consumedCount;
+        }
+
+        public long getLastConsumeTimestamp() {
+            return lastConsumeTimestamp;
+        }
+
+        public void setLastConsumeTimestamp(long lastConsumeTimestamp) {
+            this.lastConsumeTimestamp = lastConsumeTimestamp;
+        }
+
+        public long getCommitOffsetBit() {
+            return commitOffsetBit;
+        }
+
+        public void setCommitOffsetBit(long commitOffsetBit) {
+            this.commitOffsetBit = commitOffsetBit;
+        }
+
+        @JSONField(serialize = false, deserialize = false)
+        public boolean isDone() {
+            if (offsetList == null || offsetList.isEmpty()) {
+                return true;
+            }
+            int num = offsetList.size();
+            for (byte i = 0; i < num; i++) {
+                if ((commitOffsetBit & (1L << i)) == 0) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        @JSONField(serialize = false, deserialize = false)
+        public String encode() {
+            StringBuilder sb = new StringBuilder();
+            sb.append("{").append("\"c\":").append(getConsumedCount());
+            sb.append(",").append("\"cm\":").append(getCommitOffsetBit());
+            sb.append(",").append("\"l\":").append(getLastConsumeTimestamp());
+            sb.append(",").append("\"o\":[");
+            if (getOffsetList() != null) {
+                for (int i = 0; i < getOffsetList().size(); i++) {
+                    sb.append(getOffsetList().get(i));
+                    if (i < getOffsetList().size() - 1) {
+                        sb.append(",");
+                    }
+                }
+            }
+            sb.append("]").append("}");
+            return sb.toString();
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder sb = new StringBuilder("OrderInfo");
+            sb.append("@").append(this.hashCode());
+            sb.append("{offsetList=").append(offsetList);
+            sb.append(", consumedCount=").append(consumedCount);
+            sb.append(", lastConsumeTimestamp=").append(lastConsumeTimestamp);
+            sb.append(", commitOffsetBit=").append(commitOffsetBit);
+            sb.append(", isDone=").append(isDone());
+            sb.append('}');
+            return sb.toString();
+        }
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java

@@ -16,40 +16,76 @@
  */
 package org.apache.rocketmq.broker.out;
 
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
 import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
+import org.apache.rocketmq.common.BrokerIdentity;
+import org.apache.rocketmq.common.BrokerSyncInfo;
 import org.apache.rocketmq.common.DataVersion;
+import org.apache.rocketmq.common.LockCallback;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.UnlockCallback;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageBatch;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.namesrv.DefaultTopAddressing;
 import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
 import org.apache.rocketmq.common.namesrv.TopAddressing;
 import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.BrokerMemberGroup;
+import org.apache.rocketmq.common.protocol.body.ClusterInfo;
 import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
+import org.apache.rocketmq.common.protocol.body.GetBrokerMemberGroupResponseBody;
 import org.apache.rocketmq.common.protocol.body.KVTable;
+import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
+import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
+import org.apache.rocketmq.common.protocol.body.MessageRequestModeSerializeWrapper;
 import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody;
 import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
+import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper;
 import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody;
+import org.apache.rocketmq.common.protocol.header.ExchangeHAInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ExchangeHAInfoResponseHeader;
+import org.apache.rocketmq.common.protocol.header.GetBrokerMemberGroupRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader;
+import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2;
+import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.BrokerHeartbeatRequestHeader;
+import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.common.rpc.ClientMetadata;
+import org.apache.rocketmq.common.rpc.RpcClient;
+import org.apache.rocketmq.common.rpc.RpcClientImpl;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.InvokeCallback;
 import org.apache.rocketmq.remoting.RPCHook;
 import org.apache.rocketmq.remoting.RemotingClient;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
 import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
 import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException;
@@ -57,21 +93,36 @@ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
+import java.io.UnsupportedEncodingException;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
 public class BrokerOuterAPI {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final RemotingClient remotingClient;
-    private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
+    private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr());
     private String nameSrvAddr = null;
     private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
         new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
 
+    private ClientMetadata clientMetadata;
+    private RpcClient rpcClient;
+
     public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
-        this(nettyClientConfig, null);
+        this(nettyClientConfig, null, new ClientMetadata());
     }
 
-    public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) {
+    private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, ClientMetadata clientMetadata) {
         this.remotingClient = new NettyRemotingClient(nettyClientConfig);
+        this.clientMetadata = clientMetadata;
         this.remotingClient.registerRPCHook(rpcHook);
+        this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);
     }
 
     public void start() {
@@ -83,31 +134,224 @@ public class BrokerOuterAPI {
         this.brokerOuterExecutor.shutdown();
     }
 
+    public List<String> getNameServerAddressList() {
+        return this.remotingClient.getNameServerAddressList();
+    }
+
     public String fetchNameServerAddr() {
         try {
             String addrs = this.topAddressing.fetchNSAddr();
             if (addrs != null) {
                 if (!addrs.equals(this.nameSrvAddr)) {
-                    log.info("name server address changed, old: {} new: {}", this.nameSrvAddr, addrs);
+                    LOGGER.info("name server address changed, old: {} new: {}", this.nameSrvAddr, addrs);
                     this.updateNameServerAddressList(addrs);
                     this.nameSrvAddr = addrs;
                     return nameSrvAddr;
                 }
             }
         } catch (Exception e) {
-            log.error("fetchNameServerAddr Exception", e);
+            LOGGER.error("fetchNameServerAddr Exception", e);
         }
         return nameSrvAddr;
     }
 
     public void updateNameServerAddressList(final String addrs) {
-        List<String> lst = new ArrayList<String>();
         String[] addrArray = addrs.split(";");
-        for (String addr : addrArray) {
-            lst.add(addr);
+        List<String> lst = new ArrayList<String>(Arrays.asList(addrArray));
+        this.remotingClient.updateNameServerAddressList(lst);
+    }
+
+    public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+        return syncBrokerMemberGroup(clusterName, brokerName, false);
+    }
+
+    public BrokerMemberGroup syncBrokerMemberGroup(String clusterName, String brokerName,
+        boolean isCompatibleWithOldNameSrv)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+        if (isCompatibleWithOldNameSrv) {
+            return getBrokerMemberGroupCompatible(clusterName, brokerName);
+        } else {
+            return getBrokerMemberGroup(clusterName, brokerName);
+        }
+    }
+
+    public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+        BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+        GetBrokerMemberGroupRequestHeader requestHeader = new GetBrokerMemberGroupRequestHeader();
+        requestHeader.setClusterName(clusterName);
+        requestHeader.setBrokerName(brokerName);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_MEMBER_GROUP, requestHeader);
+
+        RemotingCommand response = null;
+        response = this.remotingClient.invokeSync(null, request, 3000);
+        assert response != null;
+
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                byte[] body = response.getBody();
+                if (body != null) {
+                    GetBrokerMemberGroupResponseBody brokerMemberGroupResponseBody =
+                        GetBrokerMemberGroupResponseBody.decode(body, GetBrokerMemberGroupResponseBody.class);
+
+                    return brokerMemberGroupResponseBody.getBrokerMemberGroup();
+                }
+            }
+            default:
+                break;
         }
 
-        this.remotingClient.updateNameServerAddressList(lst);
+        return brokerMemberGroup;
+    }
+
+    public BrokerMemberGroup getBrokerMemberGroupCompatible(String clusterName, String brokerName)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+        BrokerMemberGroup brokerMemberGroup = new BrokerMemberGroup(clusterName, brokerName);
+
+        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+        requestHeader.setTopic(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + brokerName);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+        RemotingCommand response;
+        response = this.remotingClient.invokeSync(null, request, 3000);
+        assert response != null;
+
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                byte[] body = response.getBody();
+                if (body != null) {
+                    TopicRouteData topicRouteData = TopicRouteData.decode(body, TopicRouteData.class);
+                    for (BrokerData brokerData : topicRouteData.getBrokerDatas()) {
+                        if (brokerData != null
+                            && brokerData.getBrokerName().equals(brokerName)
+                            && brokerData.getCluster().equals(clusterName)) {
+                            brokerMemberGroup.getBrokerAddrs().putAll(brokerData.getBrokerAddrs());
+                            break;
+                        }
+                    }
+                    return brokerMemberGroup;
+                }
+            }
+            default:
+                break;
+        }
+
+        return brokerMemberGroup;
+    }
+
+    public void sendHeartbeatViaDataVersion(
+        final String clusterName,
+        final String brokerAddr,
+        final String brokerName,
+        final Long brokerId,
+        final int timeoutMillis,
+        final DataVersion dataVersion,
+        final boolean isInBrokerContainer) {
+        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+            final QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
+            requestHeader.setBrokerAddr(brokerAddr);
+            requestHeader.setBrokerName(brokerName);
+            requestHeader.setBrokerId(brokerId);
+            requestHeader.setClusterName(clusterName);
+
+            for (final String namesrvAddr : nameServerAddressList) {
+                brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+
+                    @Override
+                    public void run2() {
+                        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
+                        request.setBody(dataVersion.encode());
+
+                        try {
+                            BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMillis);
+                        } catch (Exception e) {
+                            LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+                        }
+                    }
+                });
+            }
+        }
+    }
+
+    public void sendHeartbeat(final String clusterName,
+        final String brokerAddr,
+        final String brokerName,
+        final Long brokerId,
+        final int timeoutMills,
+        final boolean isInBrokerContainer) {
+        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
+
+        final BrokerHeartbeatRequestHeader requestHeader = new BrokerHeartbeatRequestHeader();
+        requestHeader.setClusterName(clusterName);
+        requestHeader.setBrokerAddr(brokerAddr);
+        requestHeader.setBrokerName(brokerName);
+
+        if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
+            for (final String namesrvAddr : nameServerAddressList) {
+                brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+                    @Override
+                    public void run2() {
+                        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.BROKER_HEARTBEAT, requestHeader);
+
+                        try {
+                            BrokerOuterAPI.this.remotingClient.invokeOneway(namesrvAddr, request, timeoutMills);
+                        } catch (Exception e) {
+                            LOGGER.error("sendHeartbeat Exception " + namesrvAddr, e);
+                        }
+                    }
+                });
+            }
+        }
+    }
+
+    public BrokerSyncInfo retrieveBrokerHaInfo(String masterBrokerAddr)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException,
+        MQBrokerException, RemotingCommandException {
+        ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+        requestHeader.setMasterHaAddress(null);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(masterBrokerAddr, request, 3000);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                ExchangeHAInfoResponseHeader responseHeader = (ExchangeHAInfoResponseHeader) response.decodeCommandCustomHeader(ExchangeHAInfoResponseHeader.class);
+                return new BrokerSyncInfo(responseHeader.getMasterHaAddress(), responseHeader.getMasterFlushOffset(), responseHeader.getMasterAddress());
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void sendBrokerHaInfo(String brokerAddr, String masterHaAddr, long brokerInitMaxOffset, String masterAddr)
+        throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+        ExchangeHAInfoRequestHeader requestHeader = new ExchangeHAInfoRequestHeader();
+        requestHeader.setMasterHaAddress(masterHaAddr);
+        requestHeader.setMasterFlushOffset(brokerInitMaxOffset);
+        requestHeader.setMasterAddress(masterAddr);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.EXCHANGE_BROKER_HA_INFO, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000);
+
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return;
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
     }
 
     public List<RegisterBrokerResult> registerBrokerAll(
@@ -120,10 +364,56 @@ public class BrokerOuterAPI {
         final List<String> filterServerList,
         final boolean oneway,
         final int timeoutMills,
-        final boolean compressed) {
+        final boolean enableActingMaster,
+        final boolean compressed,
+        final BrokerIdentity brokerIdentity) {
+        return registerBrokerAll(clusterName,
+            brokerAddr,
+            brokerName,
+            brokerId,
+            haServerAddr,
+            topicConfigWrapper,
+            filterServerList,
+            oneway, timeoutMills,
+            enableActingMaster,
+            compressed,
+            null,
+            brokerIdentity);
+    }
+
+    /**
+     * Considering compression brings much CPU overhead to name server, stream API will not support compression and
+     * compression feature is deprecated.
+     *
+     * @param clusterName
+     * @param brokerAddr
+     * @param brokerName
+     * @param brokerId
+     * @param haServerAddr
+     * @param topicConfigWrapper
+     * @param filterServerList
+     * @param oneway
+     * @param timeoutMills
+     * @param compressed default false
+     * @return
+     */
+    public List<RegisterBrokerResult> registerBrokerAll(
+        final String clusterName,
+        final String brokerAddr,
+        final String brokerName,
+        final long brokerId,
+        final String haServerAddr,
+        final TopicConfigSerializeWrapper topicConfigWrapper,
+        final List<String> filterServerList,
+        final boolean oneway,
+        final int timeoutMills,
+        final boolean enableActingMaster,
+        final boolean compressed,
+        final Long heartbeatTimeoutMillis,
+        final BrokerIdentity brokerIdentity) {
 
         final List<RegisterBrokerResult> registerBrokerResultList = new CopyOnWriteArrayList<>();
-        List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
+        List<String> nameServerAddressList = this.remotingClient.getAvailableNameSrvList();
         if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
 
             final RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader();
@@ -132,35 +422,43 @@ public class BrokerOuterAPI {
             requestHeader.setBrokerName(brokerName);
             requestHeader.setClusterName(clusterName);
             requestHeader.setHaServerAddr(haServerAddr);
-            requestHeader.setCompressed(compressed);
+            requestHeader.setEnableActingMaster(enableActingMaster);
+            requestHeader.setCompressed(false);
+            if (heartbeatTimeoutMillis != null) {
+                requestHeader.setHeartbeatTimeoutMillis(heartbeatTimeoutMillis);
+            }
 
             RegisterBrokerBody requestBody = new RegisterBrokerBody();
-            requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper);
+            requestBody.setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper));
             requestBody.setFilterServerList(filterServerList);
             final byte[] body = requestBody.encode(compressed);
             final int bodyCrc32 = UtilAll.crc32(body);
             requestHeader.setBodyCrc32(bodyCrc32);
             final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
             for (final String namesrvAddr : nameServerAddressList) {
-                brokerOuterExecutor.execute(() -> {
-                    try {
-                        RegisterBrokerResult result = registerBroker(namesrvAddr, oneway, timeoutMills, requestHeader, body);
-                        if (result != null) {
-                            registerBrokerResultList.add(result);
-                        }
+                brokerOuterExecutor.execute(new AbstractBrokerRunnable(brokerIdentity) {
+                    @Override public void run2() {
+                        try {
+                            RegisterBrokerResult result = registerBroker(namesrvAddr, oneway, timeoutMills, requestHeader, body);
+                            if (result != null) {
+                                registerBrokerResultList.add(result);
+                            }
 
-                        log.info("register broker[{}]to name server {} OK", brokerId, namesrvAddr);
-                    } catch (Exception e) {
-                        log.warn("registerBroker Exception, {}", namesrvAddr, e);
-                    } finally {
-                        countDownLatch.countDown();
+                            LOGGER.info("Registering current broker to name server completed. TargetHost={}", namesrvAddr);
+                        } catch (Exception e) {
+                            LOGGER.error("Failed to register current broker to name server. TargetHost={}", namesrvAddr, e);
+                        } finally {
+                            countDownLatch.countDown();
+                        }
                     }
                 });
             }
 
             try {
-                countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);
-            } catch (InterruptedException e) {
+                if (!countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS)) {
+                    LOGGER.warn("Registration to one or more name servers does NOT complete within deadline. Timeout threshold: {}ms", timeoutMills);
+                }
+            } catch (InterruptedException ignore) {
             }
         }
 
@@ -219,9 +517,9 @@ public class BrokerOuterAPI {
             for (String namesrvAddr : nameServerAddressList) {
                 try {
                     this.unregisterBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId);
-                    log.info("unregisterBroker OK, NamesrvAddr: {}", namesrvAddr);
+                    LOGGER.info("unregisterBroker OK, NamesrvAddr: {}", namesrvAddr);
                 } catch (Exception e) {
-                    log.warn("unregisterBroker Exception, {}", namesrvAddr, e);
+                    LOGGER.warn("unregisterBroker Exception, {}", namesrvAddr, e);
                 }
             }
         }
@@ -260,49 +558,52 @@ public class BrokerOuterAPI {
         final String brokerName,
         final long brokerId,
         final TopicConfigSerializeWrapper topicConfigWrapper,
-        final int timeoutMills) {
+        final int timeoutMills,
+        final boolean isInBrokerContainer) {
         final List<Boolean> changedList = new CopyOnWriteArrayList<>();
         List<String> nameServerAddressList = this.remotingClient.getNameServerAddressList();
         if (nameServerAddressList != null && nameServerAddressList.size() > 0) {
             final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size());
             for (final String namesrvAddr : nameServerAddressList) {
-                brokerOuterExecutor.execute(() -> {
-                    try {
-                        QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
-                        requestHeader.setBrokerAddr(brokerAddr);
-                        requestHeader.setBrokerId(brokerId);
-                        requestHeader.setBrokerName(brokerName);
-                        requestHeader.setClusterName(clusterName);
-                        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
-                        request.setBody(topicConfigWrapper.getDataVersion().encode());
-                        RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills);
-                        DataVersion nameServerDataVersion = null;
-                        Boolean changed = false;
-                        switch (response.getCode()) {
-                            case ResponseCode.SUCCESS: {
-                                QueryDataVersionResponseHeader queryDataVersionResponseHeader =
-                                    (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
-                                changed = queryDataVersionResponseHeader.getChanged();
-                                byte[] body = response.getBody();
-                                if (body != null) {
-                                    nameServerDataVersion = DataVersion.decode(body, DataVersion.class);
-                                    if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) {
-                                        changed = true;
+                brokerOuterExecutor.execute(new AbstractBrokerRunnable(new BrokerIdentity(clusterName, brokerName, brokerId, isInBrokerContainer)) {
+                    @Override public void run2() {
+                        try {
+                            QueryDataVersionRequestHeader requestHeader = new QueryDataVersionRequestHeader();
+                            requestHeader.setBrokerAddr(brokerAddr);
+                            requestHeader.setBrokerId(brokerId);
+                            requestHeader.setBrokerName(brokerName);
+                            requestHeader.setClusterName(clusterName);
+                            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_DATA_VERSION, requestHeader);
+                            request.setBody(topicConfigWrapper.getDataVersion().encode());
+                            RemotingCommand response = remotingClient.invokeSync(namesrvAddr, request, timeoutMills);
+                            DataVersion nameServerDataVersion = null;
+                            Boolean changed = false;
+                            switch (response.getCode()) {
+                                case ResponseCode.SUCCESS: {
+                                    QueryDataVersionResponseHeader queryDataVersionResponseHeader =
+                                        (QueryDataVersionResponseHeader) response.decodeCommandCustomHeader(QueryDataVersionResponseHeader.class);
+                                    changed = queryDataVersionResponseHeader.getChanged();
+                                    byte[] body = response.getBody();
+                                    if (body != null) {
+                                        nameServerDataVersion = DataVersion.decode(body, DataVersion.class);
+                                        if (!topicConfigWrapper.getDataVersion().equals(nameServerDataVersion)) {
+                                            changed = true;
+                                        }
+                                    }
+                                    if (changed == null || changed) {
+                                        changedList.add(Boolean.TRUE);
                                     }
                                 }
-                                if (changed == null || changed) {
-                                    changedList.add(Boolean.TRUE);
-                                }
+                                default:
+                                    break;
                             }
-                            default:
-                                break;
+                            LOGGER.warn("Query data version from name server {} OK, changed {}, broker {},name server {}", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? "" : nameServerDataVersion);
+                        } catch (Exception e) {
+                            changedList.add(Boolean.TRUE);
+                            LOGGER.error("Query data version from name server {}  Exception, {}", namesrvAddr, e);
+                        } finally {
+                            countDownLatch.countDown();
                         }
-                        log.warn("Query data version from name server {} OK,changed {}, broker {},name server {}", namesrvAddr, changed, topicConfigWrapper.getDataVersion(), nameServerDataVersion == null ? "" : nameServerDataVersion);
-                    } catch (Exception e) {
-                        changedList.add(Boolean.TRUE);
-                        log.error("Query data version from name server {}  Exception, {}", namesrvAddr, e);
-                    } finally {
-                        countDownLatch.countDown();
                     }
                 });
 
@@ -310,13 +611,13 @@ public class BrokerOuterAPI {
             try {
                 countDownLatch.await(timeoutMills, TimeUnit.MILLISECONDS);
             } catch (InterruptedException e) {
-                log.error("query dataversion from nameserver countDownLatch await Exception", e);
+                LOGGER.error("query dataversion from nameserver countDownLatch await Exception", e);
             }
         }
         return changedList;
     }
 
-    public TopicConfigSerializeWrapper getAllTopicConfig(
+    public TopicConfigAndMappingSerializeWrapper getAllTopicConfig(
         final String addr) throws RemotingConnectException, RemotingSendRequestException,
         RemotingTimeoutException, InterruptedException, MQBrokerException {
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
@@ -325,7 +626,7 @@ public class BrokerOuterAPI {
         assert response != null;
         switch (response.getCode()) {
             case ResponseCode.SUCCESS: {
-                return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class);
+                return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigAndMappingSerializeWrapper.class);
             }
             default:
                 break;
@@ -388,4 +689,303 @@ public class BrokerOuterAPI {
     public void registerRPCHook(RPCHook rpcHook) {
         remotingClient.registerRPCHook(rpcHook);
     }
+
+    public void clearRPCHook() {
+        remotingClient.clearRPCHook();
+    }
+
+    public long getMaxOffset(final String addr, final String topic, final int queueId, final boolean committed,
+        final boolean isOnlyThisBroker)
+        throws RemotingException, MQBrokerException, InterruptedException {
+        GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(queueId);
+        requestHeader.setCommitted(committed);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
+
+                return responseHeader.getOffset();
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public long getMinOffset(final String addr, final String topic, final int queueId, final boolean isOnlyThisBroker)
+        throws RemotingException, MQBrokerException, InterruptedException {
+        GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(queueId);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
+
+                return responseHeader.getOffset();
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void lockBatchMQAsync(
+        final String addr,
+        final LockBatchRequestBody requestBody,
+        final long timeoutMillis,
+        final LockCallback callback) throws RemotingException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null);
+
+        request.setBody(requestBody.encode());
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+            if (callback == null) {
+                return;
+            }
+
+            try {
+                RemotingCommand response = responseFuture.getResponseCommand();
+                if (response != null) {
+                    if (response.getCode() == ResponseCode.SUCCESS) {
+                        LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(),
+                            LockBatchResponseBody.class);
+                        Set<MessageQueue> messageQueues = responseBody.getLockOKMQSet();
+                        callback.onSuccess(messageQueues);
+                    } else {
+                        callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+                    }
+                }
+            } catch (Throwable ignored) {
+
+            }
+        });
+    }
+
+    public void unlockBatchMQAsync(
+        final String addr,
+        final UnlockBatchRequestBody requestBody,
+        final long timeoutMillis,
+        final UnlockCallback callback) throws RemotingException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null);
+
+        request.setBody(requestBody.encode());
+
+        this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> {
+            if (callback == null) {
+                return;
+            }
+
+            try {
+                RemotingCommand response = responseFuture.getResponseCommand();
+                if (response != null) {
+                    if (response.getCode() == ResponseCode.SUCCESS) {
+                        callback.onSuccess();
+                    } else {
+                        callback.onException(new MQBrokerException(response.getCode(), response.getRemark()));
+                    }
+                }
+            } catch (Throwable ignored) {
+
+            }
+        });
+    }
+
+    public RemotingClient getRemotingClient() {
+        return this.remotingClient;
+    }
+
+    public SendResult sendMessageToSpecificBroker(String brokerAddr, final String brokerName,
+        final MessageExt msg, String group,
+        long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException {
+
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        requestHeader.setProducerGroup(group);
+        requestHeader.setTopic(msg.getTopic());
+        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
+        requestHeader.setDefaultTopicQueueNums(8);
+        requestHeader.setQueueId(msg.getQueueId());
+        requestHeader.setSysFlag(msg.getSysFlag());
+        requestHeader.setBornTimestamp(msg.getBornTimestamp());
+        requestHeader.setFlag(msg.getFlag());
+        requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties()));
+        requestHeader.setReconsumeTimes(msg.getReconsumeTimes());
+        requestHeader.setBatch(false);
+
+        SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader);
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2);
+
+        request.setBody(msg.getBody());
+
+        RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
+
+        return this.processSendResponse(brokerName, msg, response);
+    }
+
+    private SendResult processSendResponse(
+        final String brokerName,
+        final Message msg,
+        final RemotingCommand response
+    ) throws MQBrokerException, RemotingCommandException {
+        switch (response.getCode()) {
+            case ResponseCode.FLUSH_DISK_TIMEOUT:
+            case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+            case ResponseCode.SLAVE_NOT_AVAILABLE:
+            case ResponseCode.SUCCESS: {
+                SendStatus sendStatus = SendStatus.SEND_OK;
+                switch (response.getCode()) {
+                    case ResponseCode.FLUSH_DISK_TIMEOUT:
+                        sendStatus = SendStatus.FLUSH_DISK_TIMEOUT;
+                        break;
+                    case ResponseCode.FLUSH_SLAVE_TIMEOUT:
+                        sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT;
+                        break;
+                    case ResponseCode.SLAVE_NOT_AVAILABLE:
+                        sendStatus = SendStatus.SLAVE_NOT_AVAILABLE;
+                        break;
+                    case ResponseCode.SUCCESS:
+                        sendStatus = SendStatus.SEND_OK;
+                        break;
+                    default:
+                        assert false;
+                        break;
+                }
+
+                SendMessageResponseHeader responseHeader =
+                    (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class);
+
+                //If namespace not null , reset Topic without namespace.
+                String topic = msg.getTopic();
+
+                MessageQueue messageQueue = new MessageQueue(topic, brokerName, responseHeader.getQueueId());
+
+                String uniqMsgId = MessageClientIDSetter.getUniqID(msg);
+                if (msg instanceof MessageBatch) {
+                    StringBuilder sb = new StringBuilder();
+                    for (Message message : (MessageBatch) msg) {
+                        sb.append(sb.length() == 0 ? "" : ",").append(MessageClientIDSetter.getUniqID(message));
+                    }
+                    uniqMsgId = sb.toString();
+                }
+                SendResult sendResult = new SendResult(sendStatus,
+                    uniqMsgId,
+                    responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset());
+                sendResult.setTransactionId(responseHeader.getTransactionId());
+                String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION);
+                String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH);
+                if (regionId == null || regionId.isEmpty()) {
+                    regionId = MixAll.DEFAULT_TRACE_REGION_ID;
+                }
+                if (traceOn != null && traceOn.equals("false")) {
+                    sendResult.setTraceOn(false);
+                } else {
+                    sendResult.setTraceOn(true);
+                }
+                sendResult.setRegionId(regionId);
+                return sendResult;
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() {
+        return brokerOuterExecutor;
+    }
+
+    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis)
+        throws RemotingException, MQBrokerException, InterruptedException {
+        return getTopicRouteInfoFromNameServer(topic, timeoutMillis, true);
+    }
+
+    public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,
+        boolean allowTopicNotExist) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
+        GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
+        requestHeader.setTopic(topic);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
+
+        RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.TOPIC_NOT_EXIST: {
+                if (allowTopicNotExist) {
+                    LOGGER.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic);
+                }
+
+                break;
+            }
+            case ResponseCode.SUCCESS: {
+                byte[] body = response.getBody();
+                if (body != null) {
+                    return TopicRouteData.decode(body, TopicRouteData.class);
+                }
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public ClusterInfo getBrokerClusterInfo() throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null);
+        RemotingCommand response = this.remotingClient.invokeSync(null, request, 3_000);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return ClusterInfo.decode(response.getBody(), ClusterInfo.class);
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark());
+    }
+
+    public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis,
+        InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException {
+        this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback);
+    }
+
+    public void refreshMetadata() throws Exception {
+        ClusterInfo brokerClusterInfo = getBrokerClusterInfo();
+        clientMetadata.refreshClusterInfo(brokerClusterInfo);
+    }
+
+    public ClientMetadata getClientMetadata() {
+        return clientMetadata;
+    }
+
+    public RpcClient getRpcClient() {
+        return rpcClient;
+    }
+
+    public MessageRequestModeSerializeWrapper getAllMessageRequestMode(
+        final String addr) throws RemotingSendRequestException, RemotingConnectException,
+        MQBrokerException, RemotingTimeoutException, InterruptedException {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_MESSAGE_REQUEST_MODE, null);
+        RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000);
+        assert response != null;
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS: {
+                return MessageRequestModeSerializeWrapper.decode(response.getBody(), MessageRequestModeSerializeWrapper.class);
+            }
+            default:
+                break;
+        }
+
+        throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/pagecache/OneMessageTransfer.java

@@ -70,6 +70,7 @@ public class OneMessageTransfer extends AbstractReferenceCounted implements File
         return 0;
     }
 
+
     @Override
     public FileRegion retain() {
         super.retain();

broker/src/main/java/org/apache/rocketmq/broker/plugin/AbstractPluginMessageStore.java

@@ -17,22 +17,43 @@
 
 package org.apache.rocketmq.broker.plugin;
 
+import java.nio.ByteBuffer;
 import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.Optional;
 import java.util.Set;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import org.apache.rocketmq.common.SystemClock;
+import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.common.protocol.body.HARuntimeInfo;
+import org.apache.rocketmq.store.AllocateMappedFileService;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.CommitLog;
 import org.apache.rocketmq.store.CommitLogDispatcher;
-import org.apache.rocketmq.store.ConsumeQueue;
+import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.QueryMessageResult;
+import org.apache.rocketmq.store.RunningFlags;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.StoreCheckpoint;
+import org.apache.rocketmq.store.StoreStatsService;
+import org.apache.rocketmq.store.TransientStorePool;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.ha.HAService;
+import org.apache.rocketmq.store.hook.PutMessageHook;
+import org.apache.rocketmq.store.hook.SendMessageBackHook;
+import org.apache.rocketmq.store.logfile.MappedFile;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.ConsumeQueueStore;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.apache.rocketmq.store.util.PerfCounter;
 
 public abstract class AbstractPluginMessageStore implements MessageStore {
     protected MessageStore next = null;
@@ -109,6 +130,11 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
         return next.getMaxOffsetInQueue(topic, queueId);
     }
 
+    @Override
+    public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) {
+        return next.getMaxOffsetInQueue(topic, queueId, committed);
+    }
+
     @Override
     public long getMinOffsetInQueue(String topic, int queueId) {
         return next.getMinOffsetInQueue(topic, queueId);
@@ -195,16 +221,6 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
         return next.queryMessage(topic, key, maxNum, begin, end);
     }
 
-    @Override
-    public void updateHaMasterAddress(String newAddr) {
-        next.updateHaMasterAddress(newAddr);
-    }
-
-    @Override
-    public long slaveFallBehindMuch() {
-        return next.slaveFallBehindMuch();
-    }
-
     @Override
     public long now() {
         return next.now();
@@ -256,7 +272,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
     }
 
     @Override
-    public ConsumeQueue getConsumeQueue(String topic, int queueId) {
+    public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {
         return next.getConsumeQueue(topic, queueId);
     }
 
@@ -266,7 +282,253 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
     }
 
     @Override
-    public void cleanUnusedLmqTopic(String topic) {
-        next.cleanUnusedLmqTopic(topic);
+    public int remainTransientStoreBufferNumbs() {
+        return next.remainTransientStoreBufferNumbs();
+    }
+
+    @Override
+    public long remainHowManyDataToCommit() {
+        return next.remainHowManyDataToCommit();
+    }
+
+    @Override
+    public long remainHowManyDataToFlush() {
+        return next.remainHowManyDataToFlush();
+    }
+
+    @Override
+    public DispatchRequest checkMessageAndReturnSize(final ByteBuffer byteBuffer, final boolean checkCRC,
+        final boolean checkDupInfo, final boolean readBody) {
+        return next.checkMessageAndReturnSize(byteBuffer, checkCRC, checkDupInfo, readBody);
+    }
+
+    @Override
+    public long getStateMachineVersion() {
+        return next.getStateMachineVersion();
+    }
+
+    @Override public PutMessageResult putMessages(MessageExtBatch messageExtBatch) {
+        return next.putMessages(messageExtBatch);
+    }
+
+    @Override public HARuntimeInfo getHARuntimeInfo() {
+        return next.getHARuntimeInfo();
+    }
+
+    @Override public boolean getLastMappedFile(long startOffset) {
+        return next.getLastMappedFile(startOffset);
+    }
+
+    @Override public void updateHaMasterAddress(String newAddr) {
+        next.updateHaMasterAddress(newAddr);
+    }
+
+    @Override public void updateMasterAddress(String newAddr) {
+        next.updateMasterAddress(newAddr);
+    }
+
+    @Override public long slaveFallBehindMuch() {
+        return next.slaveFallBehindMuch();
+    }
+
+    @Override public long getFlushedWhere() {
+        return next.getFlushedWhere();
+    }
+
+    @Override public MessageStore getMasterStoreInProcess() {
+        return next.getMasterStoreInProcess();
+    }
+
+    @Override public void setMasterStoreInProcess(MessageStore masterStoreInProcess) {
+        next.setMasterStoreInProcess(masterStoreInProcess);
+    }
+
+    @Override
+    public boolean getData(long offset, int size, ByteBuffer byteBuffer) {
+        return next.getData(offset, size, byteBuffer);
+    }
+
+    @Override public void setAliveReplicaNumInGroup(int aliveReplicaNums) {
+        next.setAliveReplicaNumInGroup(aliveReplicaNums);
+    }
+
+    @Override public int getAliveReplicaNumInGroup() {
+        return next.getAliveReplicaNumInGroup();
+    }
+
+    @Override public void wakeupHAClient() {
+        next.wakeupHAClient();
+    }
+
+    @Override public long getMasterFlushedOffset() {
+        return next.getMasterFlushedOffset();
+    }
+
+    @Override public long getBrokerInitMaxOffset() {
+        return next.getBrokerInitMaxOffset();
+    }
+
+    @Override public void setMasterFlushedOffset(long masterFlushedOffset) {
+        next.setMasterFlushedOffset(masterFlushedOffset);
+    }
+
+    @Override public void setBrokerInitMaxOffset(long brokerInitMaxOffset) {
+        next.setBrokerInitMaxOffset(brokerInitMaxOffset);
+    }
+
+    @Override public byte[] calcDeltaChecksum(long from, long to) {
+        return next.calcDeltaChecksum(from, to);
+    }
+
+    @Override public HAService getHaService() {
+        return next.getHaService();
+    }
+
+    @Override public boolean truncateFiles(long offsetToTruncate) {
+        return next.truncateFiles(offsetToTruncate);
+    }
+
+    @Override public boolean isOffsetAligned(long offset) {
+        return next.isOffsetAligned(offset);
+    }
+
+    @Override public RunningFlags getRunningFlags() {
+        return next.getRunningFlags();
+    }
+
+    @Override public void setSendMessageBackHook(SendMessageBackHook sendMessageBackHook) {
+        next.setSendMessageBackHook(sendMessageBackHook);
+    }
+
+    @Override public SendMessageBackHook getSendMessageBackHook() {
+        return next.getSendMessageBackHook();
+    }
+
+    @Override
+    public GetMessageResult getMessage(String group, String topic, int queueId, long offset,
+        int maxMsgNums, int maxTotalMsgSize, MessageFilter messageFilter) {
+        return next.getMessage(group, topic, queueId, offset, maxMsgNums, maxTotalMsgSize, messageFilter);
+    }
+
+    @Override public MessageExt lookMessageByOffset(long commitLogOffset, int size) {
+        return next.lookMessageByOffset(commitLogOffset, size);
+    }
+
+    @Override public List<SelectMappedBufferResult> getBulkCommitLogData(long offset, int size) {
+        return next.getBulkCommitLogData(offset, size);
+    }
+
+    @Override
+    public void onCommitLogAppend(MessageExtBrokerInner msg, AppendMessageResult result, MappedFile commitLogFile) {
+        next.onCommitLogAppend(msg, result, commitLogFile);
+    }
+
+    @Override
+    public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
+        boolean isRecover, boolean isFileEnd) {
+        next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd);
+    }
+
+    @Override
+    public MessageStoreConfig getMessageStoreConfig() {
+        return next.getMessageStoreConfig();
+    }
+
+    @Override
+    public StoreStatsService getStoreStatsService() {
+        return next.getStoreStatsService();
+    }
+
+    @Override
+    public StoreCheckpoint getStoreCheckpoint() {
+        return next.getStoreCheckpoint();
+    }
+
+    @Override
+    public SystemClock getSystemClock() {
+        return next.getSystemClock();
+    }
+
+    @Override
+    public CommitLog getCommitLog() {
+        return next.getCommitLog();
+    }
+
+    @Override
+    public TransientStorePool getTransientStorePool() {
+        return next.getTransientStorePool();
+    }
+
+    @Override
+    public AllocateMappedFileService getAllocateMappedFileService() {
+        return next.getAllocateMappedFileService();
+    }
+
+    @Override
+    public void truncateDirtyLogicFiles(long phyOffset) {
+        next.truncateDirtyLogicFiles(phyOffset);
+    }
+
+    @Override
+    public void destroyLogics() {
+        next.destroyLogics();
+    }
+
+    @Override
+    public void unlockMappedFile(MappedFile unlockMappedFile) {
+        next.unlockMappedFile(unlockMappedFile);
+    }
+
+    @Override
+    public PerfCounter.Ticks getPerfCounter() {
+        return next.getPerfCounter();
+    }
+
+    @Override
+    public ConsumeQueueStore getQueueStore() {
+        return next.getQueueStore();
+    }
+
+    @Override
+    public boolean isSyncDiskFlush() {
+        return next.isSyncDiskFlush();
+    }
+
+    @Override
+    public boolean isSyncMaster() {
+        return next.isSyncMaster();
+    }
+
+    @Override
+    public void assignOffset(MessageExtBrokerInner msg, short messageNum) {
+        next.assignOffset(msg, messageNum);
+    }
+
+    @Override
+    public Optional<TopicConfig> getTopicConfig(String topic) {
+        return next.getTopicConfig(topic);
+    }
+
+    @Override
+    public List<PutMessageHook> getPutMessageHookList() {
+        return next.getPutMessageHookList();
+    }
+
+    @Override
+    public long getLastFileFromOffset() {
+        return next.getLastFileFromOffset();
+    }
+
+    @Override
+    public void setPhysicalOffset(long phyOffset) {
+        next.setPhysicalOffset(phyOffset);
+    }
+
+    @Override public boolean isMappedFilesEmpty() {
+        return next.isMappedFilesEmpty();
+    }
+
+    @Override public boolean isShutdown() {
+        return next.isShutdown();
     }
 }

broker/src/main/java/org/apache/rocketmq/broker/plugin/BrokerAttachedPlugin.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.plugin;
+
+import java.util.Map;
+
+public interface BrokerAttachedPlugin {
+
+    /**
+     * Get plugin name
+     *
+     * @return plugin name
+     */
+    String pluginName();
+
+    /**
+     * Load broker attached plugin.
+     *
+     * @return load success or failed
+     */
+    boolean load();
+
+    /**
+     * Start broker attached plugin.
+     */
+    void start();
+
+    /**
+     * Shutdown broker attached plugin.
+     */
+    void shutdown();
+
+    /**
+     * Sync metadata from master.
+     */
+    void syncMetadata();
+
+    /**
+     * Sync metadata reverse from slave
+     *
+     * @param brokerAddr
+     */
+    void syncMetadataReverse(String brokerAddr) throws Exception;
+
+    /**
+     * Some plugin need build runningInfo when prepare runtime info.
+     *
+     * @param runtimeInfo
+     */
+    void buildRuntimeInfo(Map<String, String> runtimeInfo);
+
+    /**
+     * Some plugin need do something when status changed. For example, brokerRole change to master or slave.
+     *
+     * @param shouldStart
+     */
+    void statusChanged(boolean shouldStart);
+
+}

broker/src/main/java/org/apache/rocketmq/broker/plugin/MessageStoreFactory.java

@@ -22,8 +22,8 @@ import java.lang.reflect.Constructor;
 import org.apache.rocketmq.store.MessageStore;
 
 public final class MessageStoreFactory {
-    public static MessageStore build(MessageStorePluginContext context, MessageStore messageStore)
-        throws IOException {
+    public final static MessageStore build(MessageStorePluginContext context,
+        MessageStore messageStore) throws IOException {
         String plugin = context.getBrokerConfig().getMessageStorePlugIn();
         if (plugin != null && plugin.trim().length() != 0) {
             String[] pluginClasses = plugin.split(",");
@@ -31,12 +31,13 @@ public final class MessageStoreFactory {
                 String pluginClass = pluginClasses[i];
                 try {
                     @SuppressWarnings("unchecked")
-                    Class<AbstractPluginMessageStore> clazz = (Class<AbstractPluginMessageStore>) Class.forName(pluginClass);
+                    Class<AbstractPluginMessageStore> clazz = (Class<AbstractPluginMessageStore>)Class.forName(pluginClass);
                     Constructor<AbstractPluginMessageStore> construct = clazz.getConstructor(MessageStorePluginContext.class, MessageStore.class);
-                    messageStore = construct.newInstance(context, messageStore);
-                } catch (Throwable e) {
-                    throw new RuntimeException(String.format(
-                        "Initialize plugin's class %s not found!", pluginClass), e);
+                    AbstractPluginMessageStore pluginMessageStore = construct.newInstance(context, messageStore);
+                    messageStore = pluginMessageStore;
+                }
+                catch (Throwable e) {
+                    throw new RuntimeException("Initialize plugin's class: " + pluginClass + " not found!", e);
                 }
             }
         }

broker/src/main/java/org/apache/rocketmq/broker/plugin/MessageStorePluginContext.java

@@ -17,25 +17,25 @@
 
 package org.apache.rocketmq.broker.plugin;
 
+import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.store.MessageArrivingListener;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 public class MessageStorePluginContext {
+    private BrokerController controller;
     private MessageStoreConfig messageStoreConfig;
     private BrokerStatsManager brokerStatsManager;
     private MessageArrivingListener messageArrivingListener;
-    private BrokerConfig brokerConfig;
 
-    public MessageStorePluginContext(MessageStoreConfig messageStoreConfig,
-        BrokerStatsManager brokerStatsManager, MessageArrivingListener messageArrivingListener,
-        BrokerConfig brokerConfig) {
+    public MessageStorePluginContext(BrokerController controller, MessageStoreConfig messageStoreConfig,
+        BrokerStatsManager brokerStatsManager, MessageArrivingListener messageArrivingListener) {
         super();
         this.messageStoreConfig = messageStoreConfig;
         this.brokerStatsManager = brokerStatsManager;
         this.messageArrivingListener = messageArrivingListener;
-        this.brokerConfig = brokerConfig;
+        this.controller = controller;
     }
 
     public MessageStoreConfig getMessageStoreConfig() {
@@ -51,7 +51,11 @@ public class MessageStorePluginContext {
     }
 
     public BrokerConfig getBrokerConfig() {
-        return brokerConfig;
+        return controller.getBrokerConfig();
+    }
+
+    public BrokerController getController() {
+        return controller;
     }
 
 }

broker/src/main/java/org/apache/rocketmq/broker/plugin/PullMessageResultHandler.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.plugin;
+
+import io.netty.channel.Channel;
+import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.MessageFilter;
+
+public interface PullMessageResultHandler {
+
+    /**
+     * Handle result of get message from store.
+     *
+     * @param getMessageResult store result
+     * @param request request
+     * @param requestHeader request header
+     * @param channel channel
+     * @param subscriptionData sub data
+     * @param subscriptionGroupConfig sub config
+     * @param brokerAllowSuspend brokerAllowSuspend
+     * @param messageFilter store message filter
+     * @param response response
+     * @return response or null
+     */
+    RemotingCommand handle(final GetMessageResult getMessageResult,
+                           final RemotingCommand request,
+                           final PullMessageRequestHeader requestHeader,
+                           final Channel channel,
+                           final SubscriptionData subscriptionData,
+                           final SubscriptionGroupConfig subscriptionGroupConfig,
+                           final boolean brokerAllowSuspend,
+                           final MessageFilter messageFilter,
+                           final RemotingCommand response);
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java

@@ -17,15 +17,26 @@
 package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.ChannelHandlerContext;
-import java.net.InetSocketAddress;
 import java.net.SocketAddress;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.ThreadLocalRandom;
 
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.mqtrace.AbortProcessException;
+import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
+import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
 import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
 import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MQVersion;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageType;
+import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
@@ -45,46 +56,315 @@ import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2;
 import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.sysflag.TopicSysFlag;
-import org.apache.rocketmq.common.utils.ChannelUtil;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
-public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor {
-    protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+public abstract class AbstractSendMessageProcessor implements NettyRequestProcessor {
+    protected static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    protected static final InternalLogger DLQ_LOG = InternalLoggerFactory.getLogger(LoggerName.DLQ_LOGGER_NAME);
+
+    protected List<ConsumeMessageHook> consumeMessageHookList;
 
     protected final static int DLQ_NUMS_PER_GROUP = 1;
-    protected final BrokerController brokerController;
+    protected final Random random = new Random(System.currentTimeMillis());
     protected final SocketAddress storeHost;
     private List<SendMessageHook> sendMessageHookList;
+    protected final BrokerController brokerController;
+
 
     public AbstractSendMessageProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
-        this.storeHost =
-            new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), brokerController
-                .getNettyServerConfig().getListenPort());
+        this.storeHost = brokerController.getStoreHost();
+    }
+
+    public void registerConsumeMessageHook(List<ConsumeMessageHook> consumeMessageHookList) {
+        this.consumeMessageHookList = consumeMessageHookList;
+    }
+
+    protected RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, final RemotingCommand request)
+        throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        final ConsumerSendMsgBackRequestHeader requestHeader =
+            (ConsumerSendMsgBackRequestHeader) request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class);
+
+        // The send back requests sent to SlaveBroker will be forwarded to the master broker beside
+        final BrokerController masterBroker = this.brokerController.peekMasterBroker();
+        if (null == masterBroker) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("no master available along with " + brokerController.getBrokerConfig().getBrokerIP1());
+            return response;
+        }
+
+        // The broker that received the request.
+        // It may be a master broker or a slave broker
+        final BrokerController currentBroker = this.brokerController;
+
+        SubscriptionGroupConfig subscriptionGroupConfig =
+            masterBroker.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());
+        if (null == subscriptionGroupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark("subscription group not exist, " + requestHeader.getGroup() + " "
+                + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));
+            return response;
+        }
+
+        BrokerConfig masterBrokerConfig = masterBroker.getBrokerConfig();
+        if (!PermName.isWriteable(masterBrokerConfig.getBrokerPermission())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("the broker[" + masterBrokerConfig.getBrokerIP1() + "] sending message is forbidden");
+            return response;
+        }
+
+        if (subscriptionGroupConfig.getRetryQueueNums() <= 0) {
+            response.setCode(ResponseCode.SUCCESS);
+            response.setRemark(null);
+            return response;
+        }
+
+        String newTopic = MixAll.getRetryTopic(requestHeader.getGroup());
+        int queueIdInt = Math.abs(this.random.nextInt() % 99999999) % subscriptionGroupConfig.getRetryQueueNums();
+
+        int topicSysFlag = 0;
+        if (requestHeader.isUnitMode()) {
+            topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
+        }
+
+        // Create retry topic to master broker
+        TopicConfig topicConfig = masterBroker.getTopicConfigManager().createTopicInSendMessageBackMethod(
+            newTopic,
+            subscriptionGroupConfig.getRetryQueueNums(),
+            PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);
+        if (null == topicConfig) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("topic[" + newTopic + "] not exist");
+            return response;
+        }
+
+        if (!PermName.isWriteable(topicConfig.getPerm())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(String.format("the topic[%s] sending message is forbidden", newTopic));
+            return response;
+        }
+
+        // Look message from the origin message store
+        MessageExt msgExt = currentBroker.getMessageStore().lookMessageByOffset(requestHeader.getOffset());
+        if (null == msgExt) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("look message by offset failed, " + requestHeader.getOffset());
+            return response;
+        }
+
+        //for logic queue
+        if (requestHeader.getOriginTopic() != null
+            && !msgExt.getTopic().equals(requestHeader.getOriginTopic())) {
+            //here just do some fence in case of some unexpected offset is income
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("look message by offset failed to check the topic name" + requestHeader.getOffset());
+            return response;
+        }
+
+        final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
+        if (null == retryTopic) {
+            MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic());
+        }
+        msgExt.setWaitStoreMsgOK(false);
+
+        int delayLevel = requestHeader.getDelayLevel();
+
+        int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
+        if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
+            Integer times = requestHeader.getMaxReconsumeTimes();
+            if (times != null) {
+                maxReconsumeTimes = times;
+            }
+        }
+
+        boolean isDLQ = false;
+        if (msgExt.getReconsumeTimes() >= maxReconsumeTimes
+            || delayLevel < 0) {
+
+            isDLQ = true;
+            newTopic = MixAll.getDLQTopic(requestHeader.getGroup());
+            queueIdInt = randomQueueId(DLQ_NUMS_PER_GROUP);
+
+            // Create DLQ topic to master broker
+            topicConfig = masterBroker.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,
+                DLQ_NUMS_PER_GROUP,
+                PermName.PERM_WRITE | PermName.PERM_READ, 0);
+
+            if (null == topicConfig) {
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setRemark("topic[" + newTopic + "] not exist");
+                return response;
+            }
+            msgExt.setDelayTimeLevel(0);
+        } else {
+            if (0 == delayLevel) {
+                delayLevel = 3 + msgExt.getReconsumeTimes();
+            }
+
+            msgExt.setDelayTimeLevel(delayLevel);
+        }
+
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        msgInner.setTopic(newTopic);
+        msgInner.setBody(msgExt.getBody());
+        msgInner.setFlag(msgExt.getFlag());
+        MessageAccessor.setProperties(msgInner, msgExt.getProperties());
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
+        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags()));
+
+        msgInner.setQueueId(queueIdInt);
+        msgInner.setSysFlag(msgExt.getSysFlag());
+        msgInner.setBornTimestamp(msgExt.getBornTimestamp());
+        msgInner.setBornHost(msgExt.getBornHost());
+        msgInner.setStoreHost(this.getStoreHost());
+        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1);
+
+        String originMsgId = MessageAccessor.getOriginMessageId(msgExt);
+        MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId);
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
+
+        boolean succeeded = false;
+
+        // Put retry topic to master message store
+        PutMessageResult putMessageResult = masterBroker.getMessageStore().putMessage(msgInner);
+        if (putMessageResult != null) {
+            String commercialOwner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+
+            switch (putMessageResult.getPutMessageStatus()) {
+                case PUT_OK:
+                    String backTopic = msgExt.getTopic();
+                    String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
+                    if (correctTopic != null) {
+                        backTopic = correctTopic;
+                    }
+                    if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msgInner.getTopic())) {
+                        masterBroker.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());
+                        masterBroker.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+                        masterBroker.getBrokerStatsManager().incQueuePutNums(msgInner.getTopic(), msgInner.getQueueId());
+                        masterBroker.getBrokerStatsManager().incQueuePutSize(msgInner.getTopic(), msgInner.getQueueId(), putMessageResult.getAppendMessageResult().getWroteBytes());
+                    }
+                    masterBroker.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic);
+
+                    if (isDLQ) {
+                        masterBroker.getBrokerStatsManager().incDLQStatValue(
+                            BrokerStatsManager.SNDBCK2DLQ_TIMES,
+                            commercialOwner,
+                            requestHeader.getGroup(),
+                            requestHeader.getOriginTopic(),
+                            BrokerStatsManager.StatsType.SEND_BACK_TO_DLQ.name(),
+                            1);
+
+                        String uniqKey = msgInner.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
+                        DLQ_LOG.info("send msg to DLQ {}, owner={}, originalTopic={}, consumerId={}, msgUniqKey={}, storeTimestamp={}",
+                            newTopic,
+                            commercialOwner,
+                            requestHeader.getOriginTopic(),
+                            requestHeader.getGroup(),
+                            uniqKey,
+                            putMessageResult.getAppendMessageResult().getStoreTimestamp());
+                    }
+
+                    response.setCode(ResponseCode.SUCCESS);
+                    response.setRemark(null);
+
+                    succeeded = true;
+                    break;
+                default:
+                    break;
+            }
+
+            if (!succeeded) {
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setRemark(putMessageResult.getPutMessageStatus().name());
+            }
+        } else {
+            if (isDLQ) {
+                String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+                String uniqKey = msgInner.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
+                DLQ_LOG.info("failed to send msg to DLQ {}, owner={}, originalTopic={}, consumerId={}, msgUniqKey={}, result={}",
+                    newTopic,
+                    owner,
+                    requestHeader.getOriginTopic(),
+                    requestHeader.getGroup(),
+                    uniqKey,
+                    putMessageResult == null ? "null" : putMessageResult.getPutMessageStatus().toString());
+            }
+
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark("putMessageResult is null");
+        }
+
+        if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) {
+            String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getGroup());
+            ConsumeMessageContext context = new ConsumeMessageContext();
+            context.setNamespace(namespace);
+            context.setTopic(requestHeader.getOriginTopic());
+            context.setConsumerGroup(requestHeader.getGroup());
+            context.setCommercialRcvStats(BrokerStatsManager.StatsType.SEND_BACK);
+            context.setCommercialRcvTimes(1);
+            context.setCommercialOwner(request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER));
+
+            context.setAccountAuthType(request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE));
+            context.setAccountOwnerParent(request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT));
+            context.setAccountOwnerSelf(request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF));
+            context.setRcvStat(isDLQ ? BrokerStatsManager.StatsType.SEND_BACK_TO_DLQ : BrokerStatsManager.StatsType.SEND_BACK);
+            context.setSuccess(succeeded);
+            context.setRcvMsgNum(1);
+            //Set msg body size 0 when sent back by consumer.
+            context.setRcvMsgSize(0);
+            context.setCommercialRcvMsgNum(succeeded ? 1 : 0);
+
+            try {
+                this.executeConsumeMessageHookAfter(context);
+            } catch (AbortProcessException e) {
+                response.setCode(e.getResponseCode());
+                response.setRemark(e.getErrorMessage());
+            }
+        }
+
+        return response;
+    }
+
+    public boolean hasConsumeMessageHook() {
+        return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty();
+    }
+
+    public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) {
+        if (hasConsumeMessageHook()) {
+            for (ConsumeMessageHook hook : this.consumeMessageHookList) {
+                try {
+                    hook.consumeMessageAfter(context);
+                } catch (Throwable e) {
+                    // Ignore
+                }
+            }
+        }
     }
 
     protected SendMessageContext buildMsgContext(ChannelHandlerContext ctx,
         SendMessageRequestHeader requestHeader) {
-        if (!this.hasSendMessageHook()) {
-            return null;
-        }
         String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());
-        SendMessageContext mqtraceContext = new SendMessageContext();
-        mqtraceContext.setProducerGroup(requestHeader.getProducerGroup());
-        mqtraceContext.setNamespace(namespace);
-        mqtraceContext.setTopic(requestHeader.getTopic());
-        mqtraceContext.setMsgProps(requestHeader.getProperties());
-        mqtraceContext.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
-        mqtraceContext.setBrokerAddr(this.brokerController.getBrokerAddr());
-        mqtraceContext.setBrokerRegionId(this.brokerController.getBrokerConfig().getRegionId());
-        mqtraceContext.setBornTimeStamp(requestHeader.getBornTimestamp());
+
+        SendMessageContext traceContext;
+        traceContext = new SendMessageContext();
+        traceContext.setNamespace(namespace);
+        traceContext.setProducerGroup(requestHeader.getProducerGroup());
+        traceContext.setTopic(requestHeader.getTopic());
+        traceContext.setMsgProps(requestHeader.getProperties());
+        traceContext.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+        traceContext.setBrokerAddr(this.brokerController.getBrokerAddr());
+        traceContext.setBrokerRegionId(this.brokerController.getBrokerConfig().getRegionId());
+        traceContext.setBornTimeStamp(requestHeader.getBornTimestamp());
+        traceContext.setRequestTimeStamp(System.currentTimeMillis());
 
         Map<String, String> properties = MessageDecoder.string2messageProperties(requestHeader.getProperties());
         String uniqueKey = properties.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
@@ -95,8 +375,14 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         if (uniqueKey == null) {
             uniqueKey = "";
         }
-        mqtraceContext.setMsgUniqueKey(uniqueKey);
-        return mqtraceContext;
+        traceContext.setMsgUniqueKey(uniqueKey);
+
+        if (properties.containsKey(MessageConst.PROPERTY_SHARDING_KEY)) {
+            traceContext.setMsgType(MessageType.Order_Msg);
+        } else {
+            traceContext.setMsgType(MessageType.Normal_Msg);
+        }
+        return traceContext;
     }
 
     public boolean hasSendMessageHook() {
@@ -107,7 +393,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         final SendMessageRequestHeader requestHeader, final byte[] body, TopicConfig topicConfig) {
         int queueIdInt = requestHeader.getQueueId();
         if (queueIdInt < 0) {
-            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();
+            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());
         }
         int sysFlag = requestHeader.getSysFlag();
 
@@ -142,19 +428,24 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
     protected RemotingCommand msgContentCheck(final ChannelHandlerContext ctx,
         final SendMessageRequestHeader requestHeader, RemotingCommand request,
         final RemotingCommand response) {
-        if (requestHeader.getTopic().length() > Byte.MAX_VALUE) {
-            log.warn("putMessage message topic length too long {}", requestHeader.getTopic().length());
+        String topic = requestHeader.getTopic();
+        if (topic.length() > Byte.MAX_VALUE) {
+            LOGGER.warn("msgContentCheck: message topic length is too long, topic={}, topic length={}, threshold={}",
+                topic, topic.length(), Byte.MAX_VALUE);
             response.setCode(ResponseCode.MESSAGE_ILLEGAL);
             return response;
         }
         if (requestHeader.getProperties() != null && requestHeader.getProperties().length() > Short.MAX_VALUE) {
-            log.warn("putMessage message properties length too long {}", requestHeader.getProperties().length());
+            LOGGER.warn(
+                "msgContentCheck: message properties length is too long, topic={}, properties length={}, threshold={}",
+                topic, requestHeader.getProperties().length(), Short.MAX_VALUE);
             response.setCode(ResponseCode.MESSAGE_ILLEGAL);
             return response;
         }
         if (request.getBody().length > DBMsgConstants.MAX_BODY_SIZE) {
-            log.warn(" topic {}  msg body size {}  from {}", requestHeader.getTopic(),
-                request.getBody().length, ChannelUtil.getRemoteIp(ctx.channel()));
+            LOGGER.warn(
+                "msgContentCheck: message body size exceeds the threshold, topic={}, body size={}, threshold={}bytes",
+                topic, request.getBody().length, DBMsgConstants.MAX_BODY_SIZE);
             response.setRemark("msg body must be less 64KB");
             response.setCode(ResponseCode.MESSAGE_ILLEGAL);
             return response;
@@ -163,7 +454,8 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
     }
 
     protected RemotingCommand msgCheck(final ChannelHandlerContext ctx,
-        final SendMessageRequestHeader requestHeader, final RemotingCommand response) {
+        final SendMessageRequestHeader requestHeader, final RemotingCommand request,
+        final RemotingCommand response) {
         if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())
             && this.brokerController.getTopicConfigManager().isOrderTopic(requestHeader.getTopic())) {
             response.setCode(ResponseCode.NO_PERMISSION);
@@ -191,7 +483,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                 }
             }
 
-            log.warn("the topic {} not exist, producer: {}", requestHeader.getTopic(), ctx.channel().remoteAddress());
+            LOGGER.warn("the topic {} not exist, producer: {}", requestHeader.getTopic(), ctx.channel().remoteAddress());
             topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageMethod(
                 requestHeader.getTopic(),
                 requestHeader.getDefaultTopic(),
@@ -223,7 +515,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                 topicConfig,
                 RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
 
-            log.warn(errorInfo);
+            LOGGER.warn(errorInfo);
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.setRemark(errorInfo);
 
@@ -236,6 +528,20 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         this.sendMessageHookList = sendMessageHookList;
     }
 
+    protected void doResponse(ChannelHandlerContext ctx, RemotingCommand request,
+        final RemotingCommand response) {
+        if (!request.isOnewayRPC()) {
+            try {
+                ctx.writeAndFlush(response);
+            } catch (Throwable e) {
+                LOGGER.error(
+                    "SendMessageProcessor finished processing the request, but failed to send response, client "
+                        + "address={}, request={}, response={}", RemotingHelper.parseChannelRemoteAddr(ctx.channel()),
+                    request.toString(), response.toString(), e);
+            }
+        }
+    }
+
     public void executeSendMessageHookBefore(final ChannelHandlerContext ctx, final RemotingCommand request,
         SendMessageContext context) {
         if (hasSendMessageHook()) {
@@ -243,8 +549,8 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                 try {
                     final SendMessageRequestHeader requestHeader = parseRequestHeader(request);
 
+                    String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());
                     if (null != requestHeader) {
-                        String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic());
                         context.setNamespace(namespace);
                         context.setProducerGroup(requestHeader.getProducerGroup());
                         context.setTopic(requestHeader.getTopic());
@@ -259,8 +565,10 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                     if (requestHeader != null) {
                         requestHeader.setProperties(context.getMsgProps());
                     }
+                } catch (AbortProcessException e) {
+                    throw e;
                 } catch (Throwable e) {
-                    // Ignore
+                    //ignore
                 }
             }
         }
@@ -291,6 +599,83 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
         return requestHeader;
     }
 
+    static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request)
+        throws RemotingCommandException {
+        SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
+        HashMap<String, String> fields = request.getExtFields();
+        if (fields == null) {
+            throw new RemotingCommandException("the ext fields is null");
+        }
+
+        String s = fields.get("a");
+        checkNotNull(s, "the custom field <a> is null");
+        r.setA(s);
+
+        s = fields.get("b");
+        checkNotNull(s, "the custom field <b> is null");
+        r.setB(s);
+
+        s = fields.get("c");
+        checkNotNull(s, "the custom field <c> is null");
+        r.setC(s);
+
+        s = fields.get("d");
+        checkNotNull(s, "the custom field <d> is null");
+        r.setD(Integer.parseInt(s));
+
+        s = fields.get("e");
+        checkNotNull(s, "the custom field <e> is null");
+        r.setE(Integer.parseInt(s));
+
+        s = fields.get("f");
+        checkNotNull(s, "the custom field <f> is null");
+        r.setF(Integer.parseInt(s));
+
+        s = fields.get("g");
+        checkNotNull(s, "the custom field <g> is null");
+        r.setG(Long.parseLong(s));
+
+        s = fields.get("h");
+        checkNotNull(s, "the custom field <h> is null");
+        r.setH(Integer.parseInt(s));
+
+        s = fields.get("i");
+        if (s != null) {
+            r.setI(s);
+        }
+
+        s = fields.get("j");
+        if (s != null) {
+            r.setJ(Integer.parseInt(s));
+        }
+
+        s = fields.get("k");
+        if (s != null) {
+            r.setK(Boolean.parseBoolean(s));
+        }
+
+        s = fields.get("l");
+        if (s != null) {
+            r.setL(Integer.parseInt(s));
+        }
+
+        s = fields.get("m");
+        if (s != null) {
+            r.setM(Boolean.parseBoolean(s));
+        }
+        return r;
+    }
+
+    private static void checkNotNull(String s, String msg) throws RemotingCommandException {
+        if (s == null) {
+            throw new RemotingCommandException(msg);
+        }
+    }
+
+    protected int randomQueueId(int writeQueueNums) {
+        return ThreadLocalRandom.current().nextInt(99999999) % writeQueueNums;
+    }
+
     public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) {
         if (hasSendMessageHook()) {
             for (SendMessageHook hook : this.sendMessageHookList) {
@@ -306,7 +691,7 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
                     }
                     hook.sendMessageAfter(context);
                 } catch (Throwable e) {
-                    // Ignore
+                    //ignore
                 }
             }
         }

broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java

@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.alibaba.fastjson.JSON;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.util.MsgUtil;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.pop.AckMsg;
+
+public class AckMessageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final BrokerController brokerController;
+    private String reviveTopic;
+    private PopReviveService[] popReviveServices;
+
+    public AckMessageProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
+        this.popReviveServices = new PopReviveService[this.brokerController.getBrokerConfig().getReviveQueueNum()];
+        for (int i = 0; i < this.brokerController.getBrokerConfig().getReviveQueueNum(); i++) {
+            this.popReviveServices[i] = new PopReviveService(brokerController, reviveTopic, i);
+            this.popReviveServices[i].setShouldRunPopRevive(brokerController.getBrokerConfig().getBrokerId() == 0);
+        }
+    }
+
+    public void startPopReviveService() {
+        for (PopReviveService popReviveService : popReviveServices) {
+            popReviveService.start();
+        }
+    }
+
+    public void shutdownPopReviveService() {
+        for (PopReviveService popReviveService : popReviveServices) {
+            popReviveService.stop();
+        }
+    }
+
+    public void setPopReviveServiceStatus(boolean shouldStart) {
+        for (PopReviveService popReviveService : popReviveServices) {
+            popReviveService.setShouldRunPopRevive(shouldStart);
+        }
+    }
+
+    public boolean isPopReviveServiceRunning() {
+        for (PopReviveService popReviveService : popReviveServices) {
+            if (popReviveService.isShouldRunPopRevive()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        return this.processRequest(ctx.channel(), request, true);
+    }
+
+    @Override public boolean rejectRequest() {
+        return false;
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
+        boolean brokerAllowSuspend) throws RemotingCommandException {
+        final AckMessageRequestHeader requestHeader = (AckMessageRequestHeader) request.decodeCommandCustomHeader(AckMessageRequestHeader.class);
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        AckMsg ackMsg = new AckMsg();
+        RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null);
+        response.setOpaque(request.getOpaque());
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.MESSAGE_ILLEGAL);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+        long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+        if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
+            String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d",
+                requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.NO_MESSAGE);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());
+
+        ackMsg.setAckOffset(requestHeader.getOffset());
+        ackMsg.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfo));
+        ackMsg.setConsumerGroup(requestHeader.getConsumerGroup());
+        ackMsg.setTopic(requestHeader.getTopic());
+        ackMsg.setQueueId(requestHeader.getQueueId());
+        ackMsg.setPopTime(ExtraInfoUtil.getPopTime(extraInfo));
+        ackMsg.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));
+
+        int rqId = ExtraInfoUtil.getReviveQid(extraInfo);
+
+        this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);
+        this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+
+        if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) {
+            // order
+            String lockKey = requestHeader.getTopic() + PopAckConstants.SPLIT
+                + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + requestHeader.getQueueId();
+            long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(),
+                requestHeader.getTopic(), requestHeader.getQueueId());
+            if (requestHeader.getOffset() < oldOffset) {
+                return response;
+            }
+            while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) {
+            }
+            try {
+                oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(),
+                        requestHeader.getTopic(), requestHeader.getQueueId());
+                if (requestHeader.getOffset() < oldOffset) {
+                    return response;
+                }
+                long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext(
+                    requestHeader.getTopic(), requestHeader.getConsumerGroup(),
+                    requestHeader.getQueueId(), requestHeader.getOffset());
+                if (nextOffset > -1) {
+                    this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
+                        requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                        requestHeader.getQueueId(),
+                        nextOffset);
+                    this.brokerController.getPopMessageProcessor().notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(),
+                        requestHeader.getQueueId());
+                } else if (nextOffset == -1) {
+                    String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s",
+                        lockKey, oldOffset, requestHeader.getOffset(), nextOffset, channel.remoteAddress());
+                    POP_LOGGER.warn(errorInfo);
+                    response.setCode(ResponseCode.MESSAGE_ILLEGAL);
+                    response.setRemark(errorInfo);
+                    return response;
+                }
+            } finally {
+                this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey);
+            }
+            return response;
+        }
+
+        if (this.brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {
+            return response;
+        }
+
+        msgInner.setTopic(reviveTopic);
+        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
+        //msgInner.setQueueId(Integer.valueOf(extraInfo[3]));
+        msgInner.setQueueId(rqId);
+        msgInner.setTags(PopAckConstants.ACK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(this.brokerController.getStoreHost());
+        msgInner.setStoreHost(this.brokerController.getStoreHost());
+        MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo));
+        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("put ack msg error:" + putMessageResult);
+        }
+        return response;
+    }
+
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java

@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.alibaba.fastjson.JSON;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.util.MsgUtil;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader;
+import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.pop.AckMsg;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+
+public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final BrokerController brokerController;
+    private final String reviveTopic;
+
+    public ChangeInvisibleTimeProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        return this.processRequest(ctx.channel(), request, true);
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request,
+        boolean brokerAllowSuspend) throws RemotingCommandException {
+        final ChangeInvisibleTimeRequestHeader requestHeader = (ChangeInvisibleTimeRequestHeader) request.decodeCommandCustomHeader(ChangeInvisibleTimeRequestHeader.class);
+        RemotingCommand response = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setOpaque(request.getOpaque());
+        final ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.readCustomHeader();
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums() || requestHeader.getQueueId() < 0) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.MESSAGE_ILLEGAL);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+        long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
+        if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
+            response.setCode(ResponseCode.NO_MESSAGE);
+            return response;
+        }
+
+        String[] extraInfo = ExtraInfoUtil.split(requestHeader.getExtraInfo());
+
+        // add new ck
+        long now = System.currentTimeMillis();
+        PutMessageResult ckResult = appendCheckPoint(requestHeader, ExtraInfoUtil.getReviveQid(extraInfo), requestHeader.getQueueId(), requestHeader.getOffset(), now, ExtraInfoUtil.getBrokerName(extraInfo));
+
+        if (ckResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && ckResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && ckResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("change Invisible, put new ck error: {}", ckResult);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            return response;
+        }
+
+        // ack old msg.
+        try {
+            ackOrigin(requestHeader, extraInfo);
+        } catch (Throwable e) {
+            POP_LOGGER.error("change Invisible, put ack msg error: {}, {}", requestHeader.getExtraInfo(), e.getMessage());
+            // cancel new ck?
+        }
+
+        responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
+        responseHeader.setPopTime(now);
+        responseHeader.setReviveQid(ExtraInfoUtil.getReviveQid(extraInfo));
+        return response;
+    }
+
+    private void ackOrigin(final ChangeInvisibleTimeRequestHeader requestHeader, String[] extraInfo) {
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        AckMsg ackMsg = new AckMsg();
+
+        ackMsg.setAckOffset(requestHeader.getOffset());
+        ackMsg.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfo));
+        ackMsg.setConsumerGroup(requestHeader.getConsumerGroup());
+        ackMsg.setTopic(requestHeader.getTopic());
+        ackMsg.setQueueId(requestHeader.getQueueId());
+        ackMsg.setPopTime(ExtraInfoUtil.getPopTime(extraInfo));
+        ackMsg.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));
+
+        int rqId = ExtraInfoUtil.getReviveQid(extraInfo);
+
+        this.brokerController.getBrokerStatsManager().incBrokerAckNums(1);
+        this.brokerController.getBrokerStatsManager().incGroupAckNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+
+        if (brokerController.getPopMessageProcessor().getPopBufferMergeService().addAk(rqId, ackMsg)) {
+            return;
+        }
+
+        msgInner.setTopic(reviveTopic);
+        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
+        msgInner.setQueueId(rqId);
+        msgInner.setTags(PopAckConstants.ACK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(this.brokerController.getStoreHost());
+        msgInner.setStoreHost(this.brokerController.getStoreHost());
+        MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ExtraInfoUtil.getPopTime(extraInfo) + ExtraInfoUtil.getInvisibleTime(extraInfo));
+        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        PutMessageResult putMessageResult = this.brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("change Invisible, put ack msg fail: {}, {}", ackMsg, putMessageResult);
+        }
+    }
+
+    private PutMessageResult appendCheckPoint(final ChangeInvisibleTimeRequestHeader requestHeader, int reviveQid,
+        int queueId, long offset, long popTime, String brokerName) {
+        // add check point msg to revive log
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        msgInner.setTopic(reviveTopic);
+        PopCheckPoint ck = new PopCheckPoint();
+        ck.setBitMap(0);
+        ck.setNum((byte) 1);
+        ck.setPopTime(popTime);
+        ck.setInvisibleTime(requestHeader.getInvisibleTime());
+        ck.getStartOffset(offset);
+        ck.setCId(requestHeader.getConsumerGroup());
+        ck.setTopic(requestHeader.getTopic());
+        ck.setQueueId((byte) queueId);
+        ck.addDiff(0);
+
+        msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
+        msgInner.setQueueId(reviveQid);
+        msgInner.setTags(PopAckConstants.CK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(this.brokerController.getStoreHost());
+        msgInner.setStoreHost(this.brokerController.getStoreHost());
+        MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ck.getReviveTime() - PopAckConstants.ackTimeInterval);
+        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genCkUniqueId(ck));
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("change Invisible , appendCheckPoint, topic {}, queueId {},reviveId {}, cid {}, startOffset {}, rt {}, result {}", requestHeader.getTopic(), queueId, reviveQid, requestHeader.getConsumerGroup(), offset,
+                ck.getReviveTime(), putMessageResult);
+        }
+
+        if (putMessageResult != null && putMessageResult.isOk()) {
+            this.brokerController.getBrokerStatsManager().incBrokerCkNums(1);
+            this.brokerController.getBrokerStatsManager().incGroupCkNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(), 1);
+        }
+
+        return putMessageResult;
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java

@@ -28,6 +28,7 @@ import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody;
 import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
 import org.apache.rocketmq.common.protocol.header.UnregisterClientResponseHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
 import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
 import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
 import org.apache.rocketmq.common.protocol.heartbeat.ProducerData;
@@ -39,11 +40,11 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-public class ClientManageProcessor extends AsyncNettyRequestProcessor {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+public class ClientManageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
 
     public ClientManageProcessor(final BrokerController brokerController) {
@@ -81,39 +82,54 @@ public class ClientManageProcessor extends AsyncNettyRequestProcessor {
             request.getVersion()
         );
 
-        for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
+        for (ConsumerData consumerData : heartbeatData.getConsumerDataSet()) {
+            //Reject the PullConsumer
+            if (brokerController.getBrokerConfig().isRejectPullConsumerEnable()) {
+                if (ConsumeType.CONSUME_ACTIVELY == consumerData.getConsumeType()) {
+                    continue;
+                }
+            }
+
+            boolean hasOrderTopicSub = false;
+
+            for (final SubscriptionData subscriptionData : consumerData.getSubscriptionDataSet()) {
+                if (this.brokerController.getTopicConfigManager().isOrderTopic(subscriptionData.getTopic())) {
+                    hasOrderTopicSub = true;
+                    break;
+                }
+            }
+
             SubscriptionGroupConfig subscriptionGroupConfig =
                 this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(
-                    data.getGroupName());
+                    consumerData.getGroupName());
             boolean isNotifyConsumerIdsChangedEnable = true;
             if (null != subscriptionGroupConfig) {
                 isNotifyConsumerIdsChangedEnable = subscriptionGroupConfig.isNotifyConsumerIdsChangedEnable();
                 int topicSysFlag = 0;
-                if (data.isUnitMode()) {
+                if (consumerData.isUnitMode()) {
                     topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
                 }
-                String newTopic = MixAll.getRetryTopic(data.getGroupName());
+                String newTopic = MixAll.getRetryTopic(consumerData.getGroupName());
                 this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(
                     newTopic,
                     subscriptionGroupConfig.getRetryQueueNums(),
-                    PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);
+                    PermName.PERM_WRITE | PermName.PERM_READ, hasOrderTopicSub, topicSysFlag);
             }
 
             boolean changed = this.brokerController.getConsumerManager().registerConsumer(
-                data.getGroupName(),
+                consumerData.getGroupName(),
                 clientChannelInfo,
-                data.getConsumeType(),
-                data.getMessageModel(),
-                data.getConsumeFromWhere(),
-                data.getSubscriptionDataSet(),
+                consumerData.getConsumeType(),
+                consumerData.getMessageModel(),
+                consumerData.getConsumeFromWhere(),
+                consumerData.getSubscriptionDataSet(),
                 isNotifyConsumerIdsChangedEnable
             );
 
             if (changed) {
-                log.info("registerConsumer info changed {} {}",
-                    data.toString(),
-                    RemotingHelper.parseChannelRemoteAddr(ctx.channel())
-                );
+                LOGGER.info(
+                    "ClientManageProcessor: registerConsumer info changed, SDK address={}, consumerData={}",
+                    RemotingHelper.parseChannelRemoteAddr(ctx.channel()), consumerData.toString());
             }
         }
 
@@ -189,7 +205,7 @@ public class ClientManageProcessor extends AsyncNettyRequestProcessor {
             try {
                 FilterFactory.INSTANCE.get(subscriptionData.getExpressionType()).compile(subscriptionData.getSubString());
             } catch (Exception e) {
-                log.warn("Client {}@{} filter message, but failed to compile expression! sub={}, error={}",
+                LOGGER.warn("Client {}@{} filter message, but failed to compile expression! sub={}, error={}",
                     requestBody.getClientId(), requestBody.getGroup(), requestBody.getSubscriptionData(), e.getMessage());
                 response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
                 response.setRemark(e.getMessage());

broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java

@@ -17,11 +17,10 @@
 package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.ChannelHandlerContext;
-import java.util.List;
+import java.util.Set;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
 import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
@@ -31,15 +30,26 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHead
 import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
 import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
 import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader;
+import org.apache.rocketmq.common.rpc.RpcClientUtils;
+import org.apache.rocketmq.common.rpc.RpcRequest;
+import org.apache.rocketmq.common.rpc.RpcResponse;
+import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
+import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+import java.util.List;
 
+import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
+
+public class ConsumerManageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
 
     public ConsumerManageProcessor(final BrokerController brokerController) {
@@ -88,11 +98,11 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
                 response.setRemark(null);
                 return response;
             } else {
-                log.warn("getAllClientId failed, {} {}", requestHeader.getConsumerGroup(),
+                LOGGER.warn("getAllClientId failed, {} {}", requestHeader.getConsumerGroup(),
                     RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
             }
         } else {
-            log.warn("getConsumerGroupInfo failed, {} {}", requestHeader.getConsumerGroup(),
+            LOGGER.warn("getConsumerGroupInfo failed, {} {}", requestHeader.getConsumerGroup(),
                 RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
         }
 
@@ -101,6 +111,37 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
         return response;
     }
 
+    public  RemotingCommand rewriteRequestForStaticTopic(final UpdateConsumerOffsetRequestHeader requestHeader, final TopicQueueMappingContext mappingContext) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+            if (!mappingContext.isLeader()) {
+                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));
+            }
+            Long globalOffset = requestHeader.getCommitOffset();
+            LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);
+            requestHeader.setQueueId(mappingItem.getQueueId());
+            requestHeader.setLo(false);
+            requestHeader.setBname(mappingItem.getBname());
+            requestHeader.setCommitOffset(mappingItem.computePhysicalQueueOffset(globalOffset));
+            //leader, let it go, do not need to rewrite the response
+            if (mappingDetail.getBname().equals(mappingItem.getBname())) {
+                return null;
+            }
+            RpcRequest rpcRequest = new RpcRequest(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader, null);
+            RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
+            if (rpcResponse.getException() != null) {
+                throw rpcResponse.getException();
+            }
+            return RpcClientUtils.createCommandForRpcResponse(rpcResponse);
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
+    }
+
+
     private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request)
         throws RemotingCommandException {
         final RemotingCommand response =
@@ -108,13 +149,121 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
         final UpdateConsumerOffsetRequestHeader requestHeader =
             (UpdateConsumerOffsetRequestHeader) request
                 .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
-        this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getConsumerGroup(),
-            requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
-        response.setCode(ResponseCode.SUCCESS);
-        response.setRemark(null);
+        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
+
+        RemotingCommand rewriteResult  =  rewriteRequestForStaticTopic(requestHeader, mappingContext);
+        if (rewriteResult != null) {
+            return rewriteResult;
+        }
+        Set<String> topicSets = this.brokerController.getTopicConfigManager().getTopicConfigTable().keySet();
+        if (topicSets.contains(requestHeader.getTopic())) {
+            this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getConsumerGroup(),
+                requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
+            response.setCode(ResponseCode.SUCCESS);
+            response.setRemark(null);
+        } else {
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark("Topic " + requestHeader.getTopic() + " not exist!");
+        }
+
         return response;
     }
 
+
+    public  RemotingCommand rewriteRequestForStaticTopic(QueryConsumerOffsetRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+            if (!mappingContext.isLeader()) {
+                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));
+            }
+            List<LogicQueueMappingItem> mappingItemList = mappingContext.getMappingItemList();
+            if (mappingItemList.size() == 1
+                    &&  mappingItemList.get(0).getLogicOffset() == 0) {
+                //as physical, just let it go
+                mappingContext.setCurrentItem(mappingItemList.get(0));
+                requestHeader.setQueueId(mappingContext.getLeaderItem().getQueueId());
+                return null;
+            }
+            //double read check
+            List<LogicQueueMappingItem> itemList = mappingContext.getMappingItemList();
+            //by default, it is -1
+            long offset = -1;
+            //double read, first from leader, then from second leader
+            for (int i = itemList.size() - 1; i >= 0; i--) {
+                LogicQueueMappingItem mappingItem = itemList.get(i);
+                mappingContext.setCurrentItem(mappingItem);
+                if (mappingItem.getBname().equals(mappingDetail.getBname())) {
+                    offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), requestHeader.getTopic(), mappingItem.getQueueId());
+                    if (offset >= 0) {
+                        break;
+                    } else {
+                        //not found
+                        continue;
+                    }
+                } else {
+                    //maybe we need to reconstruct an object
+                    requestHeader.setBname(mappingItem.getBname());
+                    requestHeader.setQueueId(mappingItem.getQueueId());
+                    requestHeader.setLo(false);
+                    requestHeader.setSetZeroIfNotFound(false);
+                    RpcRequest rpcRequest = new RpcRequest(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader, null);
+                    RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
+                    if (rpcResponse.getException() != null) {
+                        throw rpcResponse.getException();
+                    }
+                    if (rpcResponse.getCode() == ResponseCode.SUCCESS) {
+                        offset = ((QueryConsumerOffsetResponseHeader) rpcResponse.getHeader()).getOffset();
+                        break;
+                    } else if (rpcResponse.getCode() == ResponseCode.QUERY_NOT_FOUND) {
+                        continue;
+                    } else {
+                        //this should not happen
+                        throw new RuntimeException("Unknown response code " + rpcResponse.getCode());
+                    }
+                }
+            }
+            final RemotingCommand response = RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);
+            final QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.readCustomHeader();
+            if (offset >= 0) {
+                responseHeader.setOffset(offset);
+                response.setCode(ResponseCode.SUCCESS);
+                response.setRemark(null);
+            } else {
+                response.setCode(ResponseCode.QUERY_NOT_FOUND);
+                response.setRemark("Not found, maybe this group consumer boot first");
+            }
+            RemotingCommand rewriteResponseResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());
+            if (rewriteResponseResult != null) {
+                return rewriteResponseResult;
+            }
+            return response;
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
+    }
+
+
+    public  RemotingCommand rewriteResponseForStaticTopic(final QueryConsumerOffsetRequestHeader requestHeader, final QueryConsumerOffsetResponseHeader responseHeader,
+        final TopicQueueMappingContext mappingContext, final int code) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            if (code != ResponseCode.SUCCESS) {
+                return null;
+            }
+            LogicQueueMappingItem item = mappingContext.getCurrentItem();
+            responseHeader.setOffset(item.computeStaticQueueOffsetStrictly(responseHeader.getOffset()));
+            //no need to construct new object
+            return null;
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
+    }
+
     private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request)
         throws RemotingCommandException {
         final RemotingCommand response =
@@ -125,6 +274,13 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
             (QueryConsumerOffsetRequestHeader) request
                 .decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class);
 
+
+        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
+        RemotingCommand rewriteResult  = rewriteRequestForStaticTopic(requestHeader, mappingContext);
+        if (rewriteResult != null) {
+            return rewriteResult;
+        }
+
         long offset =
             this.brokerController.getConsumerOffsetManager().queryOffset(
                 requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
@@ -137,7 +293,10 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
             long minOffset =
                 this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(),
                     requestHeader.getQueueId());
-            if (minOffset <= 0
+            if (requestHeader.getSetZeroIfNotFound() != null && Boolean.FALSE.equals(requestHeader.getSetZeroIfNotFound())) {
+                response.setCode(ResponseCode.QUERY_NOT_FOUND);
+                response.setRemark("Not found, do not set to zero, maybe this group boot first");
+            } else if (minOffset <= 0
                 && !this.brokerController.getMessageStore().checkInDiskByConsumeOffset(
                 requestHeader.getTopic(), requestHeader.getQueueId(), 0)) {
                 responseHeader.setOffset(0L);
@@ -149,6 +308,11 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor {
             }
         }
 
+        RemotingCommand rewriteResponseResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());
+        if (rewriteResponseResult != null) {
+            return rewriteResponseResult;
+        }
+
         return response;
     }
 }

broker/src/main/java/org/apache/rocketmq/broker/processor/DefaultPullMessageResultHandler.java

@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.FileRegion;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.longpolling.PullRequest;
+import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;
+import org.apache.rocketmq.broker.plugin.PullMessageResultHandler;
+import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.common.sysflag.PullSysFlag;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingUtil;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageFilter;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.config.BrokerRole;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class DefaultPullMessageResultHandler implements PullMessageResultHandler {
+
+    protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    protected final BrokerController brokerController;
+
+    public DefaultPullMessageResultHandler(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    @Override
+    public RemotingCommand handle(final GetMessageResult getMessageResult,
+                                  final RemotingCommand request,
+                                  final PullMessageRequestHeader requestHeader,
+                                  final Channel channel,
+                                  final SubscriptionData subscriptionData,
+                                  final SubscriptionGroupConfig subscriptionGroupConfig,
+                                  final boolean brokerAllowSuspend,
+                                  final MessageFilter messageFilter,
+                                  RemotingCommand response) {
+
+        final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
+
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS:
+                this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                        getMessageResult.getMessageCount());
+
+                this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                        getMessageResult.getBufferTotalSize());
+
+                this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
+
+                if (!channelIsWritable(channel, requestHeader)) {
+                    getMessageResult.release();
+                    //ignore pull request
+                    return null;
+                }
+
+                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
+
+                    final long beginTimeMills = this.brokerController.getMessageStore().now();
+                    final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
+                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
+                            requestHeader.getTopic(), requestHeader.getQueueId(),
+                            (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
+                    response.setBody(r);
+                    return response;
+                } else {
+                    try {
+                        FileRegion fileRegion =
+                                new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
+                        channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
+                            @Override
+                            public void operationComplete(ChannelFuture future) throws Exception {
+                                getMessageResult.release();
+                                if (!future.isSuccess()) {
+                                    log.error("Fail to transfer messages from page cache to {}", channel.remoteAddress(), future.cause());
+                                }
+                            }
+                        });
+                    } catch (Throwable e) {
+                        log.error("Error occurred when transferring messages from page cache", e);
+                        getMessageResult.release();
+                    }
+                    return null;
+                }
+            case ResponseCode.PULL_NOT_FOUND:
+                final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag());
+                final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;
+
+                if (brokerAllowSuspend && hasSuspendFlag) {
+                    long pollingTimeMills = suspendTimeoutMillisLong;
+                    if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {
+                        pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
+                    }
+
+                    String topic = requestHeader.getTopic();
+                    long offset = requestHeader.getQueueOffset();
+                    int queueId = requestHeader.getQueueId();
+                    PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
+                            this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
+                    this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
+                    return null;
+                }
+            case ResponseCode.PULL_RETRY_IMMEDIATELY:
+                break;
+            case ResponseCode.PULL_OFFSET_MOVED:
+                if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE
+                        || this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) {
+                    MessageQueue mq = new MessageQueue();
+                    mq.setTopic(requestHeader.getTopic());
+                    mq.setQueueId(requestHeader.getQueueId());
+                    mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
+
+                    OffsetMovedEvent event = new OffsetMovedEvent();
+                    event.setConsumerGroup(requestHeader.getConsumerGroup());
+                    event.setMessageQueue(mq);
+                    event.setOffsetRequest(requestHeader.getQueueOffset());
+                    event.setOffsetNew(getMessageResult.getNextBeginOffset());
+                    log.warn(
+                            "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}",
+                            requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(),
+                            responseHeader.getSuggestWhichBrokerId());
+                } else {
+                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
+                    response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
+                    log.warn("PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}",
+                            requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(),
+                            responseHeader.getSuggestWhichBrokerId());
+                }
+
+                break;
+            default:
+                log.warn("[BUG] impossible result code of get message: {}", response.getCode());
+                assert false;
+        }
+
+        return response;
+    }
+
+    private boolean channelIsWritable(Channel channel, PullMessageRequestHeader requestHeader) {
+        if (this.brokerController.getBrokerConfig().isNetWorkFlowController()) {
+            if (!channel.isWritable()) {
+                log.warn("channel {} not writable ,cid {}", channel.remoteAddress(), requestHeader.getConsumerGroup());
+                return false;
+            }
+
+        }
+        return true;
+    }
+
+    protected byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,
+        final int queueId) {
+        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());
+
+        long storeTimestamp = 0;
+        try {
+            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
+            for (ByteBuffer bb : messageBufferList) {
+
+                byteBuffer.put(bb);
+                int sysFlag = bb.getInt(MessageDecoder.SYSFLAG_POSITION);
+//                bornhost has the IPv4 ip if the MessageSysFlag.BORNHOST_V6_FLAG bit of sysFlag is 0
+//                IPv4 host = ip(4 byte) + port(4 byte); IPv6 host = ip(16 byte) + port(4 byte)
+                int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;
+                int msgStoreTimePos = 4 // 1 TOTALSIZE
+                    + 4 // 2 MAGICCODE
+                    + 4 // 3 BODYCRC
+                    + 4 // 4 QUEUEID
+                    + 4 // 5 FLAG
+                    + 8 // 6 QUEUEOFFSET
+                    + 8 // 7 PHYSICALOFFSET
+                    + 4 // 8 SYSFLAG
+                    + 8 // 9 BORNTIMESTAMP
+                    + bornhostLength; // 10 BORNHOST
+                storeTimestamp = bb.getLong(msgStoreTimePos);
+            }
+        } finally {
+            getMessageResult.release();
+        }
+
+        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp);
+        return byteBuffer.array();
+    }
+
+    protected void generateOffsetMovedEvent(final OffsetMovedEvent event) {
+        try {
+            MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+            msgInner.setTopic(TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT);
+            msgInner.setTags(event.getConsumerGroup());
+            msgInner.setDelayTimeLevel(0);
+            msgInner.setKeys(event.getConsumerGroup());
+            msgInner.setBody(event.encode());
+            msgInner.setFlag(0);
+            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+            msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(TopicFilterType.SINGLE_TAG, msgInner.getTags()));
+
+            msgInner.setQueueId(0);
+            msgInner.setSysFlag(0);
+            msgInner.setBornTimestamp(System.currentTimeMillis());
+            msgInner.setBornHost(RemotingUtil.string2SocketAddress(this.brokerController.getBrokerAddr()));
+            msgInner.setStoreHost(msgInner.getBornHost());
+
+            msgInner.setReconsumeTimes(0);
+
+            PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+        } catch (Exception e) {
+            log.warn(String.format("generateOffsetMovedEvent Exception, %s", event.toString()), e);
+        }
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java

@@ -32,16 +32,16 @@ import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.config.BrokerRole;
 
 /**
  * EndTransaction processor: process commit and rollback message
  */
-public class EndTransactionProcessor extends AsyncNettyRequestProcessor {
+public class EndTransactionProcessor implements NettyRequestProcessor {
     private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
     private final BrokerController brokerController;
 
@@ -230,7 +230,7 @@ public class EndTransactionProcessor extends AsyncNettyRequestProcessor {
                     response.setRemark(null);
                     break;
                 // Failed
-                case CREATE_MAPEDFILE_FAILED:
+                case CREATE_MAPPED_FILE_FAILED:
                     response.setCode(ResponseCode.SYSTEM_ERROR);
                     response.setRemark("Create mapped file failed.");
                     break;
@@ -244,7 +244,7 @@ public class EndTransactionProcessor extends AsyncNettyRequestProcessor {
                     response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE);
                     response.setRemark("Service not available now.");
                     break;
-                case OS_PAGECACHE_BUSY:
+                case OS_PAGE_CACHE_BUSY:
                     response.setCode(ResponseCode.SYSTEM_ERROR);
                     response.setRemark("OS page cache busy, please try another machine");
                     break;

broker/src/main/java/org/apache/rocketmq/broker/processor/ForwardRequestProcessor.java

@@ -21,10 +21,10 @@ import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-public class ForwardRequestProcessor extends AsyncNettyRequestProcessor {
+public class ForwardRequestProcessor implements NettyRequestProcessor {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
 
     private final BrokerController brokerController;

broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java

@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.longpolling.NotificationRequest;
+import org.apache.rocketmq.common.AbstractBrokerRunnable;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.NotificationRequestHeader;
+import org.apache.rocketmq.common.protocol.header.NotificationResponseHeader;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class NotificationProcessor implements NettyRequestProcessor {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final BrokerController brokerController;
+    private Random random = new Random(System.currentTimeMillis());
+    private static final String BORN_TIME = "bornTime";
+    private ConcurrentLinkedHashMap<String, ArrayBlockingQueue<NotificationRequest>> pollingMap = new ConcurrentLinkedHashMap.Builder<String, ArrayBlockingQueue<NotificationRequest>>().maximumWeightedCapacity(100000).build();
+    private Thread checkNotificationPollingThread;
+
+    public NotificationProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.checkNotificationPollingThread = new Thread(new AbstractBrokerRunnable(brokerController.getBrokerConfig()) {
+            @Override public void run2() {
+                while (true) {
+                    if (Thread.currentThread().isInterrupted()) {
+                        break;
+                    }
+                    try {
+                        Thread.sleep(2000L);
+                        Collection<ArrayBlockingQueue<NotificationRequest>> pops = pollingMap.values();
+                        for (ArrayBlockingQueue<NotificationRequest> popQ : pops) {
+                            NotificationRequest tmPopRequest = popQ.peek();
+                            while (tmPopRequest != null) {
+                                if (tmPopRequest.isTimeout()) {
+                                    tmPopRequest = popQ.poll();
+                                    if (tmPopRequest == null) {
+                                        break;
+                                    }
+                                    if (!tmPopRequest.isTimeout()) {
+                                        POP_LOGGER.info("not timeout , but wakeUp Notification in advance: {}", tmPopRequest);
+                                        wakeUp(tmPopRequest, false);
+                                        break;
+                                    } else {
+                                        POP_LOGGER.info("timeout , wakeUp Notification : {}", tmPopRequest);
+                                        wakeUp(tmPopRequest, false);
+                                        tmPopRequest = popQ.peek();
+                                    }
+                                } else {
+                                    break;
+                                }
+                            }
+                        }
+                    } catch (InterruptedException e) {
+                        break;
+                    } catch (Exception e) {
+                        POP_LOGGER.error("checkNotificationPolling error", e);
+                    }
+                }
+            }
+        });
+        this.checkNotificationPollingThread.setDaemon(true);
+        this.checkNotificationPollingThread.setName("checkNotificationPolling");
+        this.checkNotificationPollingThread.start();
+    }
+
+    public void shutdown() {
+        this.checkNotificationPollingThread.interrupt();
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        request.addExtField(BORN_TIME, String.valueOf(System.currentTimeMillis()));
+        return this.processRequest(ctx.channel(), request);
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    public void notifyMessageArriving(final String topic, final int queueId) {
+        ArrayBlockingQueue<NotificationRequest> remotingCommands = pollingMap.get(KeyBuilder.buildPollingNotificationKey(topic, -1));
+        if (remotingCommands != null) {
+            List<NotificationRequest> c = new ArrayList<>();
+            remotingCommands.drainTo(c);
+            for (NotificationRequest notificationRequest : c) {
+                POP_LOGGER.info("new msg arrive , wakeUp : {}", notificationRequest);
+                wakeUp(notificationRequest, true);
+
+            }
+        }
+        remotingCommands = pollingMap.get(KeyBuilder.buildPollingNotificationKey(topic, queueId));
+        if (remotingCommands != null) {
+            List<NotificationRequest> c = new ArrayList<>();
+            remotingCommands.drainTo(c);
+            for (NotificationRequest notificationRequest : c) {
+                POP_LOGGER.info("new msg arrive , wakeUp : {}", notificationRequest);
+                wakeUp(notificationRequest, true);
+            }
+        }
+    }
+
+    private void wakeUp(final NotificationRequest request, final boolean hasMsg) {
+        if (request == null || !request.complete()) {
+            return;
+        }
+        if (!request.getChannel().isActive()) {
+            return;
+        }
+        Runnable run = new Runnable() {
+            @Override
+            public void run() {
+                final RemotingCommand response = NotificationProcessor.this.responseNotification(request.getChannel(), hasMsg);
+
+                if (response != null) {
+                    response.setOpaque(request.getRemotingCommand().getOpaque());
+                    response.markResponseType();
+                    try {
+                        request.getChannel().writeAndFlush(response).addListener(new ChannelFutureListener() {
+                            @Override
+                            public void operationComplete(ChannelFuture future) throws Exception {
+                                if (!future.isSuccess()) {
+                                    POP_LOGGER.error("ProcessRequestWrapper response to {} failed", future.channel().remoteAddress(), future.cause());
+                                    POP_LOGGER.error(request.toString());
+                                    POP_LOGGER.error(response.toString());
+                                }
+                            }
+                        });
+                    } catch (Throwable e) {
+                        POP_LOGGER.error("ProcessRequestWrapper process request over, but response failed", e);
+                        POP_LOGGER.error(request.toString());
+                        POP_LOGGER.error(response.toString());
+                    }
+                }
+            }
+        };
+        this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
+    }
+
+    public RemotingCommand responseNotification(final Channel channel, boolean hasMsg) {
+        RemotingCommand response = RemotingCommand.createResponseCommand(NotificationResponseHeader.class);
+        final NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.readCustomHeader();
+        responseHeader.setHasMsg(hasMsg);
+        response.setCode(ResponseCode.SUCCESS);
+        return response;
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request)
+        throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(NotificationResponseHeader.class);
+        final NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.readCustomHeader();
+        final NotificationRequestHeader requestHeader =
+            (NotificationRequestHeader) request.decodeCommandCustomHeader(NotificationRequestHeader.class);
+
+        response.setOpaque(request.getOpaque());
+
+        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(String.format("the broker[%s] peeking message is forbidden", this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (!PermName.isReadable(topicConfig.getPerm())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("the topic[" + requestHeader.getTopic() + "] peeking message is forbidden");
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
+        if (null == subscriptionGroupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark(String.format("subscription group [%s] does not exist, %s", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
+            return response;
+        }
+
+        if (!subscriptionGroupConfig.isConsumeEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
+            return response;
+        }
+        int randomQ = random.nextInt(100);
+        boolean hasMsg = false;
+        boolean needRetry = randomQ % 5 == 0;
+        if (needRetry) {
+            TopicConfig retryTopicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()));
+            if (retryTopicConfig != null) {
+                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {
+                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();
+                    hasMsg = hasMsgFromQueue(true, requestHeader, queueId);
+                }
+            }
+        }
+        if (!hasMsg && requestHeader.getQueueId() < 0) {
+            // read all queue
+            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+                int queueId = (randomQ + i) % topicConfig.getReadQueueNums();
+                hasMsg = hasMsgFromQueue(false, requestHeader, queueId);
+                if (hasMsg) {
+                    break;
+                }
+            }
+        } else {
+            int queueId = requestHeader.getQueueId();
+            hasMsg = hasMsgFromQueue(false, requestHeader, queueId);
+        }
+
+        if (!hasMsg) {
+            if (polling(channel, request, requestHeader)) {
+                return null;
+            }
+        }
+        response.setCode(ResponseCode.SUCCESS);
+        responseHeader.setHasMsg(hasMsg);
+        return response;
+    }
+
+    private boolean hasMsgFromQueue(boolean isRetry, NotificationRequestHeader requestHeader, int queueId) {
+        String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()) : requestHeader.getTopic();
+        long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);
+        long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset;
+        return restNum > 0;
+    }
+
+    private long getPopOffset(String topic, String cid, int queueId) {
+        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(cid, topic, queueId);
+        if (offset < 0) {
+            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+        }
+        long bufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService()
+            .getLatestOffset(topic, cid, queueId);
+        if (bufferOffset < 0) {
+            return offset;
+        } else {
+            return bufferOffset > offset ? bufferOffset : offset;
+        }
+    }
+
+    private boolean polling(final Channel channel, RemotingCommand remotingCommand,
+        final NotificationRequestHeader requestHeader) {
+        if (requestHeader.getPollTime() <= 0) {
+            return false;
+        }
+
+        long expired = requestHeader.getBornTime() + requestHeader.getPollTime();
+        final NotificationRequest request = new NotificationRequest(remotingCommand, channel, expired);
+        boolean result = false;
+        if (!request.isTimeout()) {
+            String key = KeyBuilder.buildPollingNotificationKey(requestHeader.getTopic(), requestHeader.getQueueId());
+            ArrayBlockingQueue<NotificationRequest> queue = pollingMap.get(key);
+            if (queue == null) {
+                queue = new ArrayBlockingQueue<>(this.brokerController.getBrokerConfig().getPopPollingSize());
+                pollingMap.put(key, queue);
+                result = queue.offer(request);
+            } else {
+                result = queue.offer(request);
+            }
+        }
+        POP_LOGGER.info("polling {}, result {}", remotingCommand, result);
+        return result;
+
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java

@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.FileRegion;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PeekMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+
+public class PeekMessageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger LOG = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private final BrokerController brokerController;
+    private Random random = new Random(System.currentTimeMillis());
+
+    public PeekMessageProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        return this.processRequest(ctx.channel(), request, true);
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
+        throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
+        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
+        final PeekMessageRequestHeader requestHeader =
+            (PeekMessageRequestHeader) request.decodeCommandCustomHeader(PeekMessageRequestHeader.class);
+
+        response.setOpaque(request.getOpaque());
+
+        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(String.format("the broker[%s] peeking message is forbidden", this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            LOG.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (!PermName.isReadable(topicConfig.getPerm())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("the topic[" + requestHeader.getTopic() + "] peeking message is forbidden");
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+            LOG.warn(errorInfo);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
+        if (null == subscriptionGroupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark(String.format("subscription group [%s] does not exist, %s", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
+            return response;
+        }
+
+        if (!subscriptionGroupConfig.isConsumeEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
+            return response;
+        }
+        int randomQ = random.nextInt(100);
+        int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum();
+        int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg();
+        GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg);
+        boolean needRetry = randomQ % 5 == 0;
+        long popTime = System.currentTimeMillis();
+        long restNum = 0;
+        if (needRetry) {
+            TopicConfig retryTopicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()));
+            if (retryTopicConfig != null) {
+                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {
+                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();
+                    restNum = peekMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);
+                }
+            }
+        }
+        if (requestHeader.getQueueId() < 0) {
+            // read all queue
+            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+                int queueId = (randomQ + i) % topicConfig.getReadQueueNums();
+                restNum = peekMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);
+            }
+        } else {
+            int queueId = requestHeader.getQueueId();
+            restNum = peekMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);
+        }
+        // if not full , fetch retry again
+        if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums()) {
+            TopicConfig retryTopicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()));
+            if (retryTopicConfig != null) {
+                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {
+                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();
+                    restNum = peekMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime);
+                }
+            }
+        }
+        if (!getMessageResult.getMessageBufferList().isEmpty()) {
+            response.setCode(ResponseCode.SUCCESS);
+            getMessageResult.setStatus(GetMessageStatus.FOUND);
+        } else {
+            response.setCode(ResponseCode.PULL_NOT_FOUND);
+            getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
+
+        }
+        responseHeader.setRestNum(restNum);
+        response.setRemark(getMessageResult.getStatus().name());
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS:
+
+                this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                    getMessageResult.getMessageCount());
+
+                this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
+                    getMessageResult.getBufferTotalSize());
+
+                this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
+                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
+                    final long beginTimeMills = this.brokerController.getMessageStore().now();
+                    final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
+                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
+                        requestHeader.getTopic(), requestHeader.getQueueId(),
+                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
+                    response.setBody(r);
+                } else {
+                    final GetMessageResult tmpGetMessageResult = getMessageResult;
+                    try {
+                        FileRegion fileRegion =
+                            new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
+                        channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
+                            @Override
+                            public void operationComplete(ChannelFuture future) throws Exception {
+                                tmpGetMessageResult.release();
+                                if (!future.isSuccess()) {
+                                    LOG.error("Fail to transfer messages from page cache to {}", channel.remoteAddress(), future.cause());
+                                }
+                            }
+                        });
+                    } catch (Throwable e) {
+                        LOG.error("Error occurred when transferring messages from page cache", e);
+                        getMessageResult.release();
+                    }
+
+                    response = null;
+                }
+                break;
+            default:
+                assert false;
+        }
+        return response;
+    }
+
+    private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,
+        PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel,
+        long popTime) {
+        String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()) : requestHeader.getTopic();
+        GetMessageResult getMessageTmpResult;
+        long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);
+        restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+        if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
+            return restNum;
+        }
+        getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, queueId, offset,
+            requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), null);
+        // maybe store offset is not correct.
+        if (GetMessageStatus.OFFSET_TOO_SMALL.equals(getMessageTmpResult.getStatus()) || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(getMessageTmpResult.getStatus())) {
+            offset = getMessageTmpResult.getNextBeginOffset();
+            getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, queueId, offset,
+                requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), null);
+        }
+        if (getMessageTmpResult != null) {
+            for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) {
+                getMessageResult.addMessage(mapedBuffer);
+            }
+        }
+        return restNum;
+    }
+
+    private long getPopOffset(String topic, String cid, int queueId) {
+        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(cid, topic, queueId);
+        if (offset < 0) {
+            offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+        }
+        long bufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService()
+            .getLatestOffset(topic, cid, queueId);
+        if (bufferOffset < 0) {
+            return offset;
+        } else {
+            return bufferOffset > offset ? bufferOffset : offset;
+        }
+    }
+
+    private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,
+        final int queueId) {
+        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());
+
+        long storeTimestamp = 0;
+        try {
+            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
+            for (ByteBuffer bb : messageBufferList) {
+
+                byteBuffer.put(bb);
+                storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION);
+            }
+        } finally {
+            getMessageResult.release();
+        }
+
+        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp);
+        return byteBuffer.array();
+    }
+
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PollingInfoProcessor.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.concurrent.ConcurrentSkipListSet;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.longpolling.PopRequest;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PollingInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.header.PollingInfoResponseHeader;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class PollingInfoProcessor implements NettyRequestProcessor {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final BrokerController brokerController;
+
+    public PollingInfoProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        return this.processRequest(ctx.channel(), request);
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request)
+        throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(PollingInfoResponseHeader.class);
+        final PollingInfoResponseHeader responseHeader = (PollingInfoResponseHeader) response.readCustomHeader();
+        final PollingInfoRequestHeader requestHeader =
+            (PollingInfoRequestHeader) request.decodeCommandCustomHeader(PollingInfoRequestHeader.class);
+
+        response.setOpaque(request.getOpaque());
+
+        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(String.format("the broker[%s] peeking message is forbidden", this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+
+        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (!PermName.isReadable(topicConfig.getPerm())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("the topic[" + requestHeader.getTopic() + "] peeking message is forbidden");
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        SubscriptionGroupConfig subscriptionGroupConfig = this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
+        if (null == subscriptionGroupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark(String.format("subscription group [%s] does not exist, %s", requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
+            return response;
+        }
+
+        if (!subscriptionGroupConfig.isConsumeEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
+            return response;
+        }
+        String key = KeyBuilder.buildPollingKey(requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueId());
+        ConcurrentSkipListSet<PopRequest> queue = this.brokerController.getPopMessageProcessor().getPollingMap().get(key);
+        if (queue != null) {
+            responseHeader.setPollingNum(queue.size());
+        } else {
+            responseHeader.setPollingNum(0);
+        }
+        response.setCode(ResponseCode.SUCCESS);
+        return response;
+    }
+
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java

@@ -0,0 +1,776 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.alibaba.fastjson.JSON;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.util.MsgUtil;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.pop.AckMsg;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+
+public class PopBufferMergeService extends ServiceThread {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    ConcurrentHashMap<String/*mergeKey*/, PopCheckPointWrapper>
+        buffer = new ConcurrentHashMap<>(1024 * 16);
+    ConcurrentHashMap<String/*topic@cid@queueId*/, QueueWithTime<PopCheckPointWrapper>> commitOffsets =
+        new ConcurrentHashMap<>();
+    private volatile boolean serving = true;
+    private AtomicInteger counter = new AtomicInteger(0);
+    private int scanTimes = 0;
+    private final BrokerController brokerController;
+    private final PopMessageProcessor popMessageProcessor;
+    private final PopMessageProcessor.QueueLockManager queueLockManager;
+    private final long interval = 5;
+    private final long minute5 = 5 * 60 * 1000;
+    private final int countOfMinute1 = (int) (60 * 1000 / interval);
+    private final int countOfSecond1 = (int) (1000 / interval);
+    private final int countOfSecond30 = (int) (30 * 1000 / interval);
+
+    private volatile boolean master = false;
+
+    public PopBufferMergeService(BrokerController brokerController, PopMessageProcessor popMessageProcessor) {
+        this.brokerController = brokerController;
+        this.popMessageProcessor = popMessageProcessor;
+        this.queueLockManager = popMessageProcessor.getQueueLockManager();
+    }
+
+    private boolean isShouldRunning() {
+        if (this.brokerController.getBrokerConfig().isEnableSlaveActingMaster()) {
+            return true;
+        }
+        this.master = brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;
+        return this.master;
+    }
+
+    @Override
+    public String getServiceName() {
+        if (this.brokerController != null && this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return brokerController.getBrokerIdentity().getLoggerIdentifier() + PopBufferMergeService.class.getSimpleName();
+        }
+        return PopBufferMergeService.class.getSimpleName();
+    }
+
+    @Override
+    public void run() {
+        // scan
+        while (!this.isStopped()) {
+            try {
+                if (!isShouldRunning()) {
+                    // slave
+                    this.waitForRunning(interval * 200 * 5);
+                    POP_LOGGER.info("Broker is {}, {}, clear all data",
+                        brokerController.getMessageStoreConfig().getBrokerRole(), this.master);
+                    this.buffer.clear();
+                    this.commitOffsets.clear();
+                    continue;
+                }
+
+                scan();
+                if (scanTimes % countOfSecond30 == 0) {
+                    scanGarbage();
+                }
+
+                this.waitForRunning(interval);
+
+                if (!this.serving && this.buffer.size() == 0 && totalSize() == 0) {
+                    this.serving = true;
+                }
+            } catch (Throwable e) {
+                POP_LOGGER.error("PopBufferMergeService error", e);
+                this.waitForRunning(3000);
+            }
+        }
+
+        this.serving = false;
+        try {
+            Thread.sleep(2000);
+        } catch (InterruptedException e) {
+        }
+        if (!isShouldRunning()) {
+            return;
+        }
+        while (this.buffer.size() > 0 || totalSize() > 0) {
+            scan();
+        }
+    }
+
+    private int scanCommitOffset() {
+        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = this.commitOffsets.entrySet().iterator();
+        int count = 0;
+        while (iterator.hasNext()) {
+            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();
+            LinkedBlockingDeque<PopCheckPointWrapper> queue = entry.getValue().get();
+            PopCheckPointWrapper pointWrapper;
+            while ((pointWrapper = queue.peek()) != null) {
+                // 1. just offset & stored, not processed by scan
+                // 2. ck is buffer(acked)
+                // 3. ck is buffer(not all acked), all ak are stored and ck is stored
+                if ((pointWrapper.isJustOffset() && pointWrapper.isCkStored()) || isCkDone(pointWrapper)
+                    || (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored())) {
+                    if (commitOffset(pointWrapper)) {
+                        queue.poll();
+                    } else {
+                        break;
+                    }
+                } else {
+                    if (System.currentTimeMillis() - pointWrapper.getCk().getPopTime()
+                        > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2) {
+                        POP_LOGGER.warn("[PopBuffer] ck offset long time not commit, {}", pointWrapper);
+                    }
+                    break;
+                }
+            }
+            final int qs = queue.size();
+            count += qs;
+            if (qs > 5000 && scanTimes % countOfSecond1 == 0) {
+                POP_LOGGER.info("[PopBuffer] offset queue size too long, {}, {}",
+                    entry.getKey(), qs);
+            }
+        }
+        return count;
+    }
+
+    public long getLatestOffset(String lockKey) {
+        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(lockKey);
+        if (queue == null) {
+            return -1;
+        }
+        PopCheckPointWrapper pointWrapper = queue.get().peekLast();
+        if (pointWrapper != null) {
+            return pointWrapper.getNextBeginOffset();
+        }
+        return -1;
+    }
+
+    public long getLatestOffset(String topic, String group, int queueId) {
+        return getLatestOffset(KeyBuilder.buildPollingKey(topic, group, queueId));
+    }
+
+    private void scanGarbage() {
+        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = commitOffsets.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();
+            if (entry.getKey() == null) {
+                continue;
+            }
+            String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT);
+            if (keyArray == null || keyArray.length != 3) {
+                continue;
+            }
+            String topic = keyArray[0];
+            String cid = keyArray[1];
+            if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
+                POP_LOGGER.info("[PopBuffer]remove not exit topic {} in buffer!", topic);
+                iterator.remove();
+                continue;
+            }
+            if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
+                POP_LOGGER.info("[PopBuffer]remove not exit sub {} of topic {} in buffer!", cid, topic);
+                iterator.remove();
+                continue;
+            }
+            if (System.currentTimeMillis() - entry.getValue().getTime() > minute5) {
+                POP_LOGGER.info("[PopBuffer]remove long time not used sub {} of topic {} in buffer!", cid, topic);
+                iterator.remove();
+                continue;
+            }
+        }
+    }
+
+    private void scan() {
+        long startTime = System.currentTimeMillis();
+        int count = 0, countCk = 0;
+        Iterator<Map.Entry<String, PopCheckPointWrapper>> iterator = buffer.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, PopCheckPointWrapper> entry = iterator.next();
+            PopCheckPointWrapper pointWrapper = entry.getValue();
+
+            // just process offset(already stored at pull thread), or buffer ck(not stored and ack finish)
+            if ((pointWrapper.isJustOffset() && pointWrapper.isCkStored()) || isCkDone(pointWrapper)
+                || (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored())) {
+                if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                    POP_LOGGER.info("[PopBuffer]ck done, {}", pointWrapper);
+                }
+                iterator.remove();
+                counter.decrementAndGet();
+                continue;
+            }
+
+            PopCheckPoint point = pointWrapper.getCk();
+            long now = System.currentTimeMillis();
+
+            boolean removeCk = this.serving ? false : true;
+            // ck will be timeout
+            if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut()) {
+                removeCk = true;
+            }
+
+            // the time stayed is too long
+            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime()) {
+                removeCk = true;
+            }
+
+            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() * 2L) {
+                POP_LOGGER.warn("[PopBuffer]ck finish fail, stay too long, {}", pointWrapper);
+            }
+
+            // double check
+            if (isCkDone(pointWrapper)) {
+                continue;
+            } else if (pointWrapper.isJustOffset()) {
+                // just offset should be in store.
+                if (pointWrapper.getReviveQueueOffset() < 0) {
+                    putCkToStore(pointWrapper, false);
+                    countCk++;
+                }
+                continue;
+            } else if (removeCk) {
+                // put buffer ak to store
+                if (pointWrapper.getReviveQueueOffset() < 0) {
+                    putCkToStore(pointWrapper, false);
+                    countCk++;
+                }
+
+                if (!pointWrapper.isCkStored()) {
+                    continue;
+                }
+
+                for (byte i = 0; i < point.getNum(); i++) {
+                    // reput buffer ak to store
+                    if (DataConverter.getBit(pointWrapper.getBits().get(), i)
+                        && !DataConverter.getBit(pointWrapper.getToStoreBits().get(), i)) {
+                        if (putAckToStore(pointWrapper, i)) {
+                            count++;
+                            markBitCAS(pointWrapper.getToStoreBits(), i);
+                        }
+                    }
+                }
+
+                if (isCkDoneForFinish(pointWrapper) && pointWrapper.isCkStored()) {
+                    if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                        POP_LOGGER.info("[PopBuffer]ck finish, {}", pointWrapper);
+                    }
+                    iterator.remove();
+                    counter.decrementAndGet();
+                    continue;
+                }
+            }
+        }
+
+        int offsetBufferSize = scanCommitOffset();
+
+        long eclipse = System.currentTimeMillis() - startTime;
+        if (eclipse > brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() - 1000) {
+            POP_LOGGER.warn("[PopBuffer]scan stop, because eclipse too long, PopBufferEclipse={}, " +
+                    "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
+                eclipse, count, countCk, counter.get(), offsetBufferSize);
+            this.serving = false;
+        } else {
+            if (scanTimes % countOfSecond1 == 0) {
+                POP_LOGGER.info("[PopBuffer]scan, PopBufferEclipse={}, " +
+                        "PopBufferToStoreAck={}, PopBufferToStoreCk={}, PopBufferSize={}, PopBufferOffsetSize={}",
+                    eclipse, count, countCk, counter.get(), offsetBufferSize);
+            }
+        }
+        scanTimes++;
+
+        if (scanTimes >= countOfMinute1) {
+            counter.set(this.buffer.size());
+            scanTimes = 0;
+        }
+    }
+
+    private int totalSize() {
+        int count = 0;
+        Iterator<Map.Entry<String, QueueWithTime<PopCheckPointWrapper>>> iterator = this.commitOffsets.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, QueueWithTime<PopCheckPointWrapper>> entry = iterator.next();
+            LinkedBlockingDeque<PopCheckPointWrapper> queue = entry.getValue().get();
+            count += queue.size();
+        }
+        return count;
+    }
+
+    private void markBitCAS(AtomicInteger setBits, int index) {
+        while (true) {
+            int bits = setBits.get();
+            if (DataConverter.getBit(bits, index)) {
+                break;
+            }
+
+            int newBits = DataConverter.setBit(bits, index, true);
+            if (setBits.compareAndSet(bits, newBits)) {
+                break;
+            }
+        }
+    }
+
+    private boolean commitOffset(final PopCheckPointWrapper wrapper) {
+        if (wrapper.getNextBeginOffset() < 0) {
+            return true;
+        }
+
+        final PopCheckPoint popCheckPoint = wrapper.getCk();
+        final String lockKey = wrapper.getLockKey();
+
+        if (!queueLockManager.tryLock(lockKey)) {
+            return false;
+        }
+        try {
+            final long offset = brokerController.getConsumerOffsetManager().queryOffset(popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId());
+            if (wrapper.getNextBeginOffset() > offset) {
+                if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                    POP_LOGGER.info("Commit offset, {}, {}", wrapper, offset);
+                }
+            } else {
+                // maybe store offset is not correct.
+                POP_LOGGER.warn("Commit offset, consumer offset less than store, {}, {}", wrapper, offset);
+            }
+            brokerController.getConsumerOffsetManager().commitOffset(getServiceName(),
+                popCheckPoint.getCId(), popCheckPoint.getTopic(), popCheckPoint.getQueueId(), wrapper.getNextBeginOffset());
+        } finally {
+            queueLockManager.unLock(lockKey);
+        }
+        return true;
+    }
+
+    private boolean putOffsetQueue(PopCheckPointWrapper pointWrapper) {
+        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(pointWrapper.getLockKey());
+        if (queue == null) {
+            queue = new QueueWithTime<>();
+            QueueWithTime old = this.commitOffsets.putIfAbsent(pointWrapper.getLockKey(), queue);
+            if (old != null) {
+                queue = old;
+            }
+        }
+        queue.setTime(pointWrapper.getCk().getPopTime());
+        return queue.get().offer(pointWrapper);
+    }
+
+    private boolean checkQueueOk(PopCheckPointWrapper pointWrapper) {
+        QueueWithTime<PopCheckPointWrapper> queue = this.commitOffsets.get(pointWrapper.getLockKey());
+        if (queue == null) {
+            return true;
+        }
+        return queue.get().size() < brokerController.getBrokerConfig().getPopCkOffsetMaxQueueSize();
+    }
+
+    /**
+     * put to store && add to buffer.
+     *
+     * @param point
+     * @param reviveQueueId
+     * @param reviveQueueOffset
+     * @param nextBeginOffset
+     * @return
+     */
+    public void addCkJustOffset(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) {
+        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset, true);
+
+        this.putCkToStore(pointWrapper, !checkQueueOk(pointWrapper));
+
+        putOffsetQueue(pointWrapper);
+        this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);
+        this.counter.incrementAndGet();
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]add ck just offset, {}", pointWrapper);
+        }
+    }
+
+    public void addCkMock(String group, String topic, int queueId, long startOffset, long invisibleTime,
+        long popTime, int reviveQueueId, long nextBeginOffset, String brokerName) {
+        final PopCheckPoint ck = new PopCheckPoint();
+        ck.setBitMap(0);
+        ck.setNum((byte) 0);
+        ck.setPopTime(popTime);
+        ck.setInvisibleTime(invisibleTime);
+        ck.getStartOffset(startOffset);
+        ck.setCId(group);
+        ck.setTopic(topic);
+        ck.setQueueId((byte) queueId);
+        ck.setBrokerName(brokerName);
+
+        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, Long.MAX_VALUE, ck, nextBeginOffset, true);
+        pointWrapper.setCkStored(true);
+
+        putOffsetQueue(pointWrapper);
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]add ck just offset, mocked, {}", pointWrapper);
+        }
+    }
+
+    public boolean addCk(PopCheckPoint point, int reviveQueueId, long reviveQueueOffset, long nextBeginOffset) {
+        // key: point.getT() + point.getC() + point.getQ() + point.getSo() + point.getPt()
+        if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) {
+            return false;
+        }
+        if (!serving) {
+            return false;
+        }
+
+        long now = System.currentTimeMillis();
+        if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) {
+            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                POP_LOGGER.warn("[PopBuffer]add ck, timeout, {}, {}", point, now);
+            }
+            return false;
+        }
+
+        if (this.counter.get() > brokerController.getBrokerConfig().getPopCkMaxBufferSize()) {
+            POP_LOGGER.warn("[PopBuffer]add ck, max size, {}, {}", point, this.counter.get());
+            return false;
+        }
+
+        PopCheckPointWrapper pointWrapper = new PopCheckPointWrapper(reviveQueueId, reviveQueueOffset, point, nextBeginOffset);
+
+        if (!checkQueueOk(pointWrapper)) {
+            return false;
+        }
+
+        putOffsetQueue(pointWrapper);
+        this.buffer.put(pointWrapper.getMergeKey(), pointWrapper);
+        this.counter.incrementAndGet();
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]add ck, {}", pointWrapper);
+        }
+        return true;
+    }
+
+    public boolean addAk(int reviveQid, AckMsg ackMsg) {
+        if (!brokerController.getBrokerConfig().isEnablePopBufferMerge()) {
+            return false;
+        }
+        if (!serving) {
+            return false;
+        }
+        try {
+            PopCheckPointWrapper pointWrapper = this.buffer.get(ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime() + ackMsg.getBrokerName());
+            if (pointWrapper == null) {
+                if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                    POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, no ck, {}", reviveQid, ackMsg);
+                }
+                return false;
+            }
+
+            if (pointWrapper.isJustOffset()) {
+                return false;
+            }
+
+            PopCheckPoint point = pointWrapper.getCk();
+            long now = System.currentTimeMillis();
+
+            if (point.getReviveTime() - now < brokerController.getBrokerConfig().getPopCkStayBufferTimeOut() + 1500) {
+                if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                    POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, almost timeout for revive, {}, {}, {}", reviveQid, pointWrapper, ackMsg, now);
+                }
+                return false;
+            }
+
+            if (now - point.getPopTime() > brokerController.getBrokerConfig().getPopCkStayBufferTime() - 1500) {
+                if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                    POP_LOGGER.warn("[PopBuffer]add ack fail, rqId={}, stay too long, {}, {}, {}", reviveQid, pointWrapper, ackMsg, now);
+                }
+                return false;
+            }
+
+            int indexOfAck = point.indexOfAck(ackMsg.getAckOffset());
+            if (indexOfAck > -1) {
+                markBitCAS(pointWrapper.getBits(), indexOfAck);
+            } else {
+                POP_LOGGER.error("[PopBuffer]Invalid index of ack, reviveQid={}, {}, {}", reviveQid, ackMsg, point);
+                return true;
+            }
+
+            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                POP_LOGGER.info("[PopBuffer]add ack, rqId={}, {}, {}", reviveQid, pointWrapper, ackMsg);
+            }
+
+//            // check ak done
+//            if (isCkDone(pointWrapper)) {
+//                // cancel ck for timer
+//                cancelCkTimer(pointWrapper);
+//            }
+            return true;
+        } catch (Throwable e) {
+            POP_LOGGER.error("[PopBuffer]add ack error, rqId=" + reviveQid + ", " + ackMsg, e);
+        }
+
+        return false;
+    }
+
+    public void clearOffsetQueue(String lockKey) {
+        this.commitOffsets.remove(lockKey);
+    }
+
+    private void putCkToStore(final PopCheckPointWrapper pointWrapper, final boolean runInCurrent) {
+        if (pointWrapper.getReviveQueueOffset() >= 0) {
+            return;
+        }
+        MessageExtBrokerInner msgInner = popMessageProcessor.buildCkMsg(pointWrapper.getCk(), pointWrapper.getReviveQueueId());
+        PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner);
+        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("[PopBuffer]put ck to store fail: {}, {}", pointWrapper, putMessageResult);
+            return;
+        }
+        pointWrapper.setCkStored(true);
+        pointWrapper.setReviveQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]put ck to store ok: {}, {}", pointWrapper, putMessageResult);
+        }
+    }
+
+    private boolean putAckToStore(final PopCheckPointWrapper pointWrapper, byte msgIndex) {
+        PopCheckPoint point = pointWrapper.getCk();
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        final AckMsg ackMsg = new AckMsg();
+
+        ackMsg.setAckOffset(point.ackOffsetByIndex(msgIndex));
+        ackMsg.setStartOffset(point.getStartOffset());
+        ackMsg.setConsumerGroup(point.getCId());
+        ackMsg.setTopic(point.getTopic());
+        ackMsg.setQueueId(point.getQueueId());
+        ackMsg.setPopTime(point.getPopTime());
+        msgInner.setTopic(popMessageProcessor.reviveTopic);
+        msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset));
+        msgInner.setQueueId(pointWrapper.getReviveQueueId());
+        msgInner.setTags(PopAckConstants.ACK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(brokerController.getStoreHost());
+        msgInner.setStoreHost(brokerController.getStoreHost());
+
+        MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, point.getReviveTime());
+        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, PopMessageProcessor.genAckUniqueId(ackMsg));
+
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner);
+        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("[PopBuffer]put ack to store fail: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
+            return false;
+        }
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]put ack to store ok: {}, {}, {}", pointWrapper, ackMsg, putMessageResult);
+        }
+
+        return true;
+    }
+
+    private boolean cancelCkTimer(final PopCheckPointWrapper pointWrapper) {
+        // not stored, no need cancel
+        if (pointWrapper.getReviveQueueOffset() < 0) {
+            return true;
+        }
+        PopCheckPoint point = pointWrapper.getCk();
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        msgInner.setTopic(popMessageProcessor.reviveTopic);
+        msgInner.setBody((pointWrapper.getReviveQueueId() + "-" + pointWrapper.getReviveQueueOffset()).getBytes());
+        msgInner.setQueueId(pointWrapper.getReviveQueueId());
+        msgInner.setTags(PopAckConstants.CK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(brokerController.getStoreHost());
+        msgInner.setStoreHost(brokerController.getStoreHost());
+
+        MsgUtil.setMessageDeliverTime(brokerController, msgInner, point.getReviveTime() - PopAckConstants.ackTimeInterval);
+
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner);
+        if (putMessageResult.getPutMessageStatus() != PutMessageStatus.PUT_OK
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_DISK_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.FLUSH_SLAVE_TIMEOUT
+            && putMessageResult.getPutMessageStatus() != PutMessageStatus.SLAVE_NOT_AVAILABLE) {
+            POP_LOGGER.error("[PopBuffer]PutMessageCallback cancelCheckPoint fail, {}, {}", pointWrapper, putMessageResult);
+            return false;
+        }
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("[PopBuffer]cancelCheckPoint, {}", pointWrapper);
+        }
+        return true;
+    }
+
+    private boolean isCkDone(PopCheckPointWrapper pointWrapper) {
+        byte num = pointWrapper.getCk().getNum();
+        for (byte i = 0; i < num; i++) {
+            if (!DataConverter.getBit(pointWrapper.getBits().get(), i)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean isCkDoneForFinish(PopCheckPointWrapper pointWrapper) {
+        byte num = pointWrapper.getCk().getNum();
+        int bits = pointWrapper.getBits().get() ^ pointWrapper.getToStoreBits().get();
+        for (byte i = 0; i < num; i++) {
+            if (DataConverter.getBit(bits, i)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    public class QueueWithTime<T> {
+        private final LinkedBlockingDeque<T> queue;
+        private long time;
+
+        public QueueWithTime() {
+            this.queue = new LinkedBlockingDeque<>();
+            this.time = System.currentTimeMillis();
+        }
+
+        public void setTime(long popTime) {
+            this.time = popTime;
+        }
+
+        public long getTime() {
+            return time;
+        }
+
+        public LinkedBlockingDeque<T> get() {
+            return queue;
+        }
+    }
+
+    public class PopCheckPointWrapper {
+        private final int reviveQueueId;
+        // -1: not stored, >=0: stored, Long.MAX: storing.
+        private volatile long reviveQueueOffset;
+        private final PopCheckPoint ck;
+        // bit for concurrent
+        private final AtomicInteger bits;
+        // bit for stored buffer ak
+        private final AtomicInteger toStoreBits;
+        private final long nextBeginOffset;
+        private final String lockKey;
+        private final String mergeKey;
+        private final boolean justOffset;
+        private volatile boolean ckStored = false;
+
+        public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point,
+            long nextBeginOffset) {
+            this.reviveQueueId = reviveQueueId;
+            this.reviveQueueOffset = reviveQueueOffset;
+            this.ck = point;
+            this.bits = new AtomicInteger(0);
+            this.toStoreBits = new AtomicInteger(0);
+            this.nextBeginOffset = nextBeginOffset;
+            this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId();
+            this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName();
+            this.justOffset = false;
+        }
+
+        public PopCheckPointWrapper(int reviveQueueId, long reviveQueueOffset, PopCheckPoint point,
+            long nextBeginOffset,
+            boolean justOffset) {
+            this.reviveQueueId = reviveQueueId;
+            this.reviveQueueOffset = reviveQueueOffset;
+            this.ck = point;
+            this.bits = new AtomicInteger(0);
+            this.toStoreBits = new AtomicInteger(0);
+            this.nextBeginOffset = nextBeginOffset;
+            this.lockKey = ck.getTopic() + PopAckConstants.SPLIT + ck.getCId() + PopAckConstants.SPLIT + ck.getQueueId();
+            this.mergeKey = point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime() + point.getBrokerName();
+            this.justOffset = justOffset;
+        }
+
+        public int getReviveQueueId() {
+            return reviveQueueId;
+        }
+
+        public long getReviveQueueOffset() {
+            return reviveQueueOffset;
+        }
+
+        public boolean isCkStored() {
+            return ckStored;
+        }
+
+        public void setReviveQueueOffset(long reviveQueueOffset) {
+            this.reviveQueueOffset = reviveQueueOffset;
+        }
+
+        public PopCheckPoint getCk() {
+            return ck;
+        }
+
+        public AtomicInteger getBits() {
+            return bits;
+        }
+
+        public AtomicInteger getToStoreBits() {
+            return toStoreBits;
+        }
+
+        public long getNextBeginOffset() {
+            return nextBeginOffset;
+        }
+
+        public String getLockKey() {
+            return lockKey;
+        }
+
+        public String getMergeKey() {
+            return mergeKey;
+        }
+
+        public boolean isJustOffset() {
+            return justOffset;
+        }
+
+        public void setCkStored(boolean ckStored) {
+            this.ckStored = ckStored;
+        }
+
+        @Override
+        public String toString() {
+            final StringBuilder sb = new StringBuilder("CkWrap{");
+            sb.append("rq=").append(reviveQueueId);
+            sb.append(", rqo=").append(reviveQueueOffset);
+            sb.append(", ck=").append(ck);
+            sb.append(", bits=").append(bits);
+            sb.append(", sBits=").append(toStoreBits);
+            sb.append(", nbo=").append(nextBeginOffset);
+            sb.append(", cks=").append(ckStored);
+            sb.append(", jo=").append(justOffset);
+            sb.append('}');
+            return sb.toString();
+        }
+    }
+
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java

@@ -0,0 +1,983 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.alibaba.fastjson.JSON;
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelFutureListener;
+import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.FileRegion;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.filter.ConsumerFilterData;
+import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
+import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
+import org.apache.rocketmq.broker.longpolling.PopRequest;
+import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;
+import org.apache.rocketmq.broker.util.MsgUtil;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.ConsumeInitMode;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.filter.ExpressionType;
+import org.apache.rocketmq.common.filter.FilterAPI;
+import org.apache.rocketmq.common.help.FAQUrl;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.RequestTask;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.pop.AckMsg;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+
+public class PopMessageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger POP_LOGGER =
+        InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+    private final BrokerController brokerController;
+    private Random random = new Random(System.currentTimeMillis());
+    String reviveTopic;
+    private static final String BORN_TIME = "bornTime";
+
+    private static final int POLLING_SUC = 0;
+    private static final int POLLING_FULL = 1;
+    private static final int POLLING_TIMEOUT = 2;
+    private static final int NOT_POLLING = 3;
+
+    private ConcurrentHashMap<String, ConcurrentHashMap<String, Byte>> topicCidMap;
+    private ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> pollingMap;
+    private AtomicLong totalPollingNum = new AtomicLong(0);
+    private PopLongPollingService popLongPollingService;
+    private PopBufferMergeService popBufferMergeService;
+    private QueueLockManager queueLockManager;
+    private AtomicLong ckMessageNumber;
+
+    public PopMessageProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.reviveTopic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
+        // 100000 topic default,  100000 lru topic + cid + qid
+        this.topicCidMap = new ConcurrentHashMap<>(this.brokerController.getBrokerConfig().getPopPollingMapSize());
+        this.pollingMap = new ConcurrentLinkedHashMap.Builder<String, ConcurrentSkipListSet<PopRequest>>()
+            .maximumWeightedCapacity(this.brokerController.getBrokerConfig().getPopPollingMapSize()).build();
+        this.popLongPollingService = new PopLongPollingService();
+        this.queueLockManager = new QueueLockManager();
+        this.popBufferMergeService = new PopBufferMergeService(this.brokerController, this);
+        this.ckMessageNumber = new AtomicLong();
+    }
+
+    public PopLongPollingService getPopLongPollingService() {
+        return popLongPollingService;
+    }
+
+    public PopBufferMergeService getPopBufferMergeService() {
+        return this.popBufferMergeService;
+    }
+
+    public QueueLockManager getQueueLockManager() {
+        return queueLockManager;
+    }
+
+    public static String genAckUniqueId(AckMsg ackMsg) {
+        return ackMsg.getTopic()
+            + PopAckConstants.SPLIT + ackMsg.getQueueId()
+            + PopAckConstants.SPLIT + ackMsg.getAckOffset()
+            + PopAckConstants.SPLIT + ackMsg.getConsumerGroup()
+            + PopAckConstants.SPLIT + ackMsg.getPopTime()
+            + PopAckConstants.SPLIT + ackMsg.getBrokerName()
+            + PopAckConstants.SPLIT + PopAckConstants.ACK_TAG;
+    }
+
+    public static String genCkUniqueId(PopCheckPoint ck) {
+        return ck.getTopic()
+            + PopAckConstants.SPLIT + ck.getQueueId()
+            + PopAckConstants.SPLIT + ck.getStartOffset()
+            + PopAckConstants.SPLIT + ck.getCId()
+            + PopAckConstants.SPLIT + ck.getPopTime()
+            + PopAckConstants.SPLIT + ck.getBrokerName()
+            + PopAckConstants.SPLIT + PopAckConstants.CK_TAG;
+    }
+
+    @Override
+    public RemotingCommand processRequest(final ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        request.addExtField(BORN_TIME, String.valueOf(System.currentTimeMillis()));
+        return this.processRequest(ctx.channel(), request);
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    public ConcurrentLinkedHashMap<String, ConcurrentSkipListSet<PopRequest>> getPollingMap() {
+        return pollingMap;
+    }
+
+    public void notifyMessageArriving(final String topic, final int queueId) {
+        ConcurrentHashMap<String, Byte> cids = topicCidMap.get(topic);
+        if (cids == null) {
+            return;
+        }
+        for (Entry<String, Byte> cid : cids.entrySet()) {
+            if (queueId >= 0) {
+                notifyMessageArriving(topic, cid.getKey(), -1);
+            }
+            notifyMessageArriving(topic, cid.getKey(), queueId);
+        }
+    }
+
+    public void notifyMessageArriving(final String topic, final String cid, final int queueId) {
+        ConcurrentSkipListSet<PopRequest> remotingCommands = pollingMap.get(KeyBuilder.buildPollingKey(topic, cid, queueId));
+        if (remotingCommands == null || remotingCommands.isEmpty()) {
+            return;
+        }
+        PopRequest popRequest = remotingCommands.pollFirst();
+        //clean inactive channel
+        while (popRequest != null && !popRequest.getChannel().isActive()) {
+            totalPollingNum.decrementAndGet();
+            popRequest = remotingCommands.pollFirst();
+        }
+
+        if (popRequest == null) {
+            return;
+        }
+        totalPollingNum.decrementAndGet();
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("lock release , new msg arrive , wakeUp : {}", popRequest);
+        }
+        wakeUp(popRequest);
+    }
+
+    private void wakeUp(final PopRequest request) {
+        if (request == null || !request.complete()) {
+            return;
+        }
+        if (!request.getChannel().isActive()) {
+            return;
+        }
+        Runnable run = new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    final RemotingCommand response = PopMessageProcessor.this.processRequest(request.getChannel(), request.getRemotingCommand());
+
+                    if (response != null) {
+                        response.setOpaque(request.getRemotingCommand().getOpaque());
+                        response.markResponseType();
+                        try {
+                            request.getChannel().writeAndFlush(response).addListener(new ChannelFutureListener() {
+                                @Override
+                                public void operationComplete(ChannelFuture future) throws Exception {
+                                    if (!future.isSuccess()) {
+                                        POP_LOGGER.error("ProcessRequestWrapper response to {} failed", future.channel().remoteAddress(), future.cause());
+                                        POP_LOGGER.error(request.toString());
+                                        POP_LOGGER.error(response.toString());
+                                    }
+                                }
+                            });
+                        } catch (Throwable e) {
+                            POP_LOGGER.error("ProcessRequestWrapper process request over, but response failed", e);
+                            POP_LOGGER.error(request.toString());
+                            POP_LOGGER.error(response.toString());
+                        }
+                    }
+                } catch (RemotingCommandException e1) {
+                    POP_LOGGER.error("ExecuteRequestWhenWakeup run", e1);
+                }
+            }
+        };
+        this.brokerController.getPullMessageExecutor().submit(new RequestTask(run, request.getChannel(), request.getRemotingCommand()));
+    }
+
+    private RemotingCommand processRequest(final Channel channel, RemotingCommand request)
+        throws RemotingCommandException {
+        RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
+        final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
+        final PopMessageRequestHeader requestHeader =
+            (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class);
+        StringBuilder startOffsetInfo = new StringBuilder(64);
+        StringBuilder msgOffsetInfo = new StringBuilder(64);
+        StringBuilder orderCountInfo = null;
+        if (requestHeader.isOrder()) {
+            orderCountInfo = new StringBuilder(64);
+        }
+
+        response.setOpaque(request.getOpaque());
+
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("receive PopMessage request command, {}", request);
+        }
+
+        if (requestHeader.isTimeoutTooMuch()) {
+            response.setCode(POLLING_TIMEOUT);
+            response.setRemark(String.format("the broker[%s] poping message is timeout too much",
+                this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+        if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(String.format("the broker[%s] poping message is forbidden",
+                this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+        if (requestHeader.getMaxMsgNums() > 32) {
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(String.format("the broker[%s] poping message's num is greater than 32",
+                this.brokerController.getBrokerConfig().getBrokerIP1()));
+            return response;
+        }
+
+        TopicConfig topicConfig =
+            this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
+        if (null == topicConfig) {
+            POP_LOGGER.error("The topic {} not exist, consumer: {} ", requestHeader.getTopic(),
+                RemotingHelper.parseChannelRemoteAddr(channel));
+            response.setCode(ResponseCode.TOPIC_NOT_EXIST);
+            response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(),
+                FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
+            return response;
+        }
+
+        if (!PermName.isReadable(topicConfig.getPerm())) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("the topic[" + requestHeader.getTopic() + "] peeking message is forbidden");
+            return response;
+        }
+
+        if (requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
+            String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] " +
+                    "consumer:[%s]",
+                requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(),
+                channel.remoteAddress());
+            POP_LOGGER.warn(errorInfo);
+            response.setCode(ResponseCode.SYSTEM_ERROR);
+            response.setRemark(errorInfo);
+            return response;
+        }
+        SubscriptionGroupConfig subscriptionGroupConfig =
+            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
+        if (null == subscriptionGroupConfig) {
+            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
+            response.setRemark(String.format("subscription group [%s] does not exist, %s",
+                requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
+            return response;
+        }
+        ConsumerGroupInfo consumerGroupInfo =
+            this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
+        if (null == consumerGroupInfo) {
+            POP_LOGGER.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
+            response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
+            response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
+            return response;
+        }
+
+        if (!subscriptionGroupConfig.isConsumeEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
+            return response;
+        }
+
+        ExpressionMessageFilter messageFilter = null;
+        if (requestHeader.getExp() != null && requestHeader.getExp().length() > 0) {
+            try {
+                SubscriptionData subscriptionData = FilterAPI.build(requestHeader.getTopic(), requestHeader.getExp(), requestHeader.getExpType());
+                ConsumerFilterData consumerFilterData = null;
+                if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
+                    consumerFilterData = ConsumerFilterManager.build(
+                        requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getExp(),
+                        requestHeader.getExpType(), System.currentTimeMillis()
+                    );
+                    if (consumerFilterData == null) {
+                        POP_LOGGER.warn("Parse the consumer's subscription[{}] failed, group: {}",
+                            requestHeader.getExp(), requestHeader.getConsumerGroup());
+                        response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
+                        response.setRemark("parse the consumer's subscription failed");
+                        return response;
+                    }
+                }
+                messageFilter = new ExpressionMessageFilter(subscriptionData, consumerFilterData,
+                    brokerController.getConsumerFilterManager());
+            } catch (Exception e) {
+                POP_LOGGER.warn("Parse the consumer's subscription[{}] error, group: {}", requestHeader.getExp(),
+                    requestHeader.getConsumerGroup());
+                response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
+                response.setRemark("parse the consumer's subscription failed");
+                return response;
+            }
+        }
+
+        int randomQ = random.nextInt(100);
+        int reviveQid;
+        if (requestHeader.isOrder()) {
+            reviveQid = KeyBuilder.POP_ORDER_REVIVE_QUEUE;
+        } else {
+            reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum());
+        }
+
+        int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg();
+        GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg);
+
+        long restNum = 0;
+        boolean needRetry = randomQ % 5 == 0;
+        long popTime = System.currentTimeMillis();
+        if (needRetry && !requestHeader.isOrder()) {
+            TopicConfig retryTopicConfig =
+                this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()));
+            if (retryTopicConfig != null) {
+                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {
+                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();
+                    restNum = popMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid,
+                        channel, popTime, messageFilter,
+                        startOffsetInfo, msgOffsetInfo, orderCountInfo);
+                }
+            }
+        }
+        if (requestHeader.getQueueId() < 0) {
+            // read all queue
+            for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
+                int queueId = (randomQ + i) % topicConfig.getReadQueueNums();
+                restNum = popMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel, popTime, messageFilter,
+                    startOffsetInfo, msgOffsetInfo, orderCountInfo);
+            }
+        } else {
+            int queueId = requestHeader.getQueueId();
+            restNum = popMsgFromQueue(false, getMessageResult, requestHeader, queueId, restNum, reviveQid, channel,
+                popTime, messageFilter,
+                startOffsetInfo, msgOffsetInfo, orderCountInfo);
+        }
+        // if not full , fetch retry again
+        if (!needRetry && getMessageResult.getMessageMapedList().size() < requestHeader.getMaxMsgNums() && !requestHeader.isOrder()) {
+            TopicConfig retryTopicConfig =
+                this.brokerController.getTopicConfigManager().selectTopicConfig(KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup()));
+            if (retryTopicConfig != null) {
+                for (int i = 0; i < retryTopicConfig.getReadQueueNums(); i++) {
+                    int queueId = (randomQ + i) % retryTopicConfig.getReadQueueNums();
+                    restNum = popMsgFromQueue(true, getMessageResult, requestHeader, queueId, restNum, reviveQid,
+                        channel, popTime, messageFilter,
+                        startOffsetInfo, msgOffsetInfo, orderCountInfo);
+                }
+            }
+        }
+        if (!getMessageResult.getMessageBufferList().isEmpty()) {
+            response.setCode(ResponseCode.SUCCESS);
+            getMessageResult.setStatus(GetMessageStatus.FOUND);
+            if (restNum > 0) {
+                // all queue pop can not notify specified queue pop, and vice versa
+                notifyMessageArriving(requestHeader.getTopic(), requestHeader.getConsumerGroup(),
+                    requestHeader.getQueueId());
+            }
+        } else {
+            int pollingResult = polling(channel, request, requestHeader);
+            if (POLLING_SUC == pollingResult) {
+                return null;
+            } else if (POLLING_FULL == pollingResult) {
+                response.setCode(ResponseCode.POLLING_FULL);
+            } else {
+                response.setCode(ResponseCode.POLLING_TIMEOUT);
+            }
+            getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
+        }
+        responseHeader.setInvisibleTime(requestHeader.getInvisibleTime());
+        responseHeader.setPopTime(popTime);
+        responseHeader.setReviveQid(reviveQid);
+        responseHeader.setRestNum(restNum);
+        responseHeader.setStartOffsetInfo(startOffsetInfo.toString());
+        responseHeader.setMsgOffsetInfo(msgOffsetInfo.toString());
+        if (requestHeader.isOrder() && orderCountInfo != null) {
+            responseHeader.setOrderCountInfo(orderCountInfo.toString());
+        }
+        response.setRemark(getMessageResult.getStatus().name());
+        switch (response.getCode()) {
+            case ResponseCode.SUCCESS:
+                if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
+                    final long beginTimeMills = this.brokerController.getMessageStore().now();
+                    final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(),
+                        requestHeader.getTopic(), requestHeader.getQueueId());
+                    this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
+                        requestHeader.getTopic(), requestHeader.getQueueId(),
+                        (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
+                    response.setBody(r);
+                } else {
+                    final GetMessageResult tmpGetMessageResult = getMessageResult;
+                    try {
+                        FileRegion fileRegion =
+                            new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()),
+                                getMessageResult);
+                        channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
+                            @Override
+                            public void operationComplete(ChannelFuture future) throws Exception {
+                                tmpGetMessageResult.release();
+                                if (!future.isSuccess()) {
+                                    POP_LOGGER.error("Fail to transfer messages from page cache to {}",
+                                        channel.remoteAddress(), future.cause());
+                                }
+                            }
+                        });
+                    } catch (Throwable e) {
+                        POP_LOGGER.error("Error occurred when transferring messages from page cache", e);
+                        getMessageResult.release();
+                    }
+
+                    response = null;
+                }
+                break;
+            default:
+                assert false;
+        }
+        return response;
+    }
+
+    private long popMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,
+        PopMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid,
+        Channel channel, long popTime,
+        ExpressionMessageFilter messageFilter, StringBuilder startOffsetInfo,
+        StringBuilder msgOffsetInfo, StringBuilder orderCountInfo) {
+        String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(),
+            requestHeader.getConsumerGroup()) : requestHeader.getTopic();
+        String lockKey =
+            topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId;
+        boolean isOrder = requestHeader.isOrder();
+        long offset = getPopOffset(topic, requestHeader, queueId, false, lockKey);
+        if (!queueLockManager.tryLock(lockKey)) {
+            restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+            return restNum;
+        }
+        offset = getPopOffset(topic, requestHeader, queueId, true, lockKey);
+        GetMessageResult getMessageTmpResult;
+        try {
+            if (isOrder && brokerController.getConsumerOrderInfoManager().checkBlock(topic,
+                requestHeader.getConsumerGroup(), queueId, requestHeader.getInvisibleTime())) {
+                return this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+            }
+
+            if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
+                restNum =
+                    this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+                return restNum;
+            }
+            getMessageTmpResult = this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup()
+                    , topic, queueId, offset,
+                    requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter);
+            if (getMessageTmpResult == null) {
+                return this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
+            }
+            // maybe store offset is not correct.
+            if (GetMessageStatus.OFFSET_TOO_SMALL.equals(getMessageTmpResult.getStatus())
+                    || GetMessageStatus.OFFSET_OVERFLOW_BADLY.equals(getMessageTmpResult.getStatus())
+                    || GetMessageStatus.OFFSET_FOUND_NULL.equals(getMessageTmpResult.getStatus())) {
+                // commit offset, because the offset is not correct
+                // If offset in store is greater than cq offset, it will cause duplicate messages,
+                // because offset in PopBuffer is not committed.
+                POP_LOGGER.warn("Pop initial offset, because store is no correct, {}, {}->{}",
+                        lockKey, offset, getMessageTmpResult.getNextBeginOffset());
+                offset = getMessageTmpResult.getNextBeginOffset();
+                this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
+                    queueId, offset);
+                getMessageTmpResult =
+                    this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic,
+                        queueId, offset,
+                        requestHeader.getMaxMsgNums() - getMessageResult.getMessageMapedList().size(), messageFilter);
+            }
+
+            restNum = getMessageTmpResult.getMaxOffset() - getMessageTmpResult.getNextBeginOffset() + restNum;
+            if (!getMessageTmpResult.getMessageMapedList().isEmpty()) {
+                this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageTmpResult.getMessageCount());
+                this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), topic,
+                    getMessageTmpResult.getMessageCount());
+                this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), topic,
+                    getMessageTmpResult.getBufferTotalSize());
+
+                if (isOrder) {
+                    int count = brokerController.getConsumerOrderInfoManager().update(topic,
+                        requestHeader.getConsumerGroup(),
+                        queueId, getMessageTmpResult.getMessageQueueOffset());
+                    this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(),
+                        requestHeader.getConsumerGroup(), topic, queueId, offset);
+                    ExtraInfoUtil.buildOrderCountInfo(orderCountInfo, isRetry, queueId, count);
+                } else {
+                    appendCheckPoint(requestHeader, topic, reviveQid, queueId, offset, getMessageTmpResult, popTime, this.brokerController.getBrokerConfig().getBrokerName());
+                }
+                ExtraInfoUtil.buildStartOffsetInfo(startOffsetInfo, isRetry, queueId, offset);
+                ExtraInfoUtil.buildMsgOffsetInfo(msgOffsetInfo, isRetry, queueId,
+                    getMessageTmpResult.getMessageQueueOffset());
+            } else if ((GetMessageStatus.NO_MATCHED_MESSAGE.equals(getMessageTmpResult.getStatus())
+                || GetMessageStatus.OFFSET_FOUND_NULL.equals(getMessageTmpResult.getStatus())
+                || GetMessageStatus.MESSAGE_WAS_REMOVING.equals(getMessageTmpResult.getStatus())
+                || GetMessageStatus.NO_MATCHED_LOGIC_QUEUE.equals(getMessageTmpResult.getStatus()))
+                && getMessageTmpResult.getNextBeginOffset() > -1) {
+                popBufferMergeService.addCkMock(requestHeader.getConsumerGroup(), topic, queueId, offset,
+                    requestHeader.getInvisibleTime(), popTime, reviveQid, getMessageTmpResult.getNextBeginOffset(), brokerController.getBrokerConfig().getBrokerName());
+//                this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), requestHeader.getConsumerGroup(), topic,
+//                        queueId, getMessageTmpResult.getNextBeginOffset());
+            }
+        } finally {
+            queueLockManager.unLock(lockKey);
+        }
+        if (getMessageTmpResult != null) {
+            for (SelectMappedBufferResult mapedBuffer : getMessageTmpResult.getMessageMapedList()) {
+                getMessageResult.addMessage(mapedBuffer);
+            }
+        }
+        return restNum;
+    }
+
+    private long getPopOffset(String topic, PopMessageRequestHeader requestHeader, int queueId, boolean init,
+        String lockKey) {
+        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(),
+            topic, queueId);
+        if (offset < 0) {
+            if (ConsumeInitMode.MIN == requestHeader.getInitMode()) {
+                offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
+            } else {
+                // pop last one,then commit offset.
+                offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - 1;
+                // max & no consumer offset
+                if (offset < 0) {
+                    offset = 0;
+                }
+                if (init) {
+                    this.brokerController.getConsumerOffsetManager().commitOffset("getPopOffset",
+                        requestHeader.getConsumerGroup(), topic,
+                        queueId, offset);
+                }
+            }
+        }
+        long bufferOffset = this.popBufferMergeService.getLatestOffset(lockKey);
+        if (bufferOffset < 0) {
+            return offset;
+        } else {
+            return bufferOffset > offset ? bufferOffset : offset;
+        }
+    }
+
+    /**
+     * @param channel
+     * @param remotingCommand
+     * @param requestHeader
+     * @return
+     */
+    private int polling(final Channel channel, RemotingCommand remotingCommand,
+        final PopMessageRequestHeader requestHeader) {
+        if (requestHeader.getPollTime() <= 0 || this.popLongPollingService.isStopped()) {
+            return NOT_POLLING;
+        }
+        ConcurrentHashMap<String, Byte> cids = topicCidMap.get(requestHeader.getTopic());
+        if (cids == null) {
+            cids = new ConcurrentHashMap<>();
+            ConcurrentHashMap<String, Byte> old = topicCidMap.putIfAbsent(requestHeader.getTopic(), cids);
+            if (old != null) {
+                cids = old;
+            }
+        }
+        cids.putIfAbsent(requestHeader.getConsumerGroup(), Byte.MIN_VALUE);
+        long expired = requestHeader.getBornTime() + requestHeader.getPollTime();
+        final PopRequest request = new PopRequest(remotingCommand, channel, expired);
+        boolean isFull = totalPollingNum.get() >= this.brokerController.getBrokerConfig().getMaxPopPollingSize();
+        if (isFull) {
+            POP_LOGGER.info("polling {}, result POLLING_FULL, total:{}", remotingCommand, totalPollingNum.get());
+            return POLLING_FULL;
+        }
+        boolean isTimeout = request.isTimeout();
+        if (isTimeout) {
+            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                POP_LOGGER.info("polling {}, result POLLING_TIMEOUT", remotingCommand);
+            }
+            return POLLING_TIMEOUT;
+        }
+        String key = KeyBuilder.buildPollingKey(requestHeader.getTopic(), requestHeader.getConsumerGroup(),
+            requestHeader.getQueueId());
+        ConcurrentSkipListSet<PopRequest> queue = pollingMap.get(key);
+        if (queue == null) {
+            queue = new ConcurrentSkipListSet<>(PopRequest.COMPARATOR);
+            ConcurrentSkipListSet<PopRequest> old = pollingMap.putIfAbsent(key, queue);
+            if (old != null) {
+                queue = old;
+            }
+        } else {
+            // check size
+            int size = queue.size();
+            if (size > brokerController.getBrokerConfig().getPopPollingSize()) {
+                POP_LOGGER.info("polling {}, result POLLING_FULL, singleSize:{}", remotingCommand, size);
+                return POLLING_FULL;
+            }
+        }
+        if (queue.add(request)) {
+            totalPollingNum.incrementAndGet();
+            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                POP_LOGGER.info("polling {}, result POLLING_SUC", remotingCommand);
+            }
+            return POLLING_SUC;
+        } else {
+            POP_LOGGER.info("polling {}, result POLLING_FULL, add fail, {}", request, queue);
+            return POLLING_FULL;
+        }
+    }
+
+    public final MessageExtBrokerInner buildCkMsg(final PopCheckPoint ck, final int reviveQid) {
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+
+        msgInner.setTopic(reviveTopic);
+        msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
+        msgInner.setQueueId(reviveQid);
+        msgInner.setTags(PopAckConstants.CK_TAG);
+        msgInner.setBornTimestamp(System.currentTimeMillis());
+        msgInner.setBornHost(this.brokerController.getStoreHost());
+        msgInner.setStoreHost(this.brokerController.getStoreHost());
+        MsgUtil.setMessageDeliverTime(this.brokerController, msgInner, ck.getReviveTime() - PopAckConstants.ackTimeInterval);
+        msgInner.getProperties().put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, genCkUniqueId(ck));
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+
+        return msgInner;
+    }
+
+    private void appendCheckPoint(final PopMessageRequestHeader requestHeader,
+        final String topic, final int reviveQid, final int queueId, final long offset,
+        final GetMessageResult getMessageTmpResult, final long popTime, final String brokerName) {
+        // add check point msg to revive log
+        final PopCheckPoint ck = new PopCheckPoint();
+        ck.setBitMap(0);
+        ck.setNum((byte) getMessageTmpResult.getMessageMapedList().size());
+        ck.setPopTime(popTime);
+        ck.setInvisibleTime(requestHeader.getInvisibleTime());
+        ck.getStartOffset(offset);
+        ck.setCId(requestHeader.getConsumerGroup());
+        ck.setTopic(topic);
+        ck.setQueueId((byte) queueId);
+        ck.setBrokerName(brokerName);
+        for (Long msgQueueOffset : getMessageTmpResult.getMessageQueueOffset()) {
+            ck.addDiff((int) (msgQueueOffset - offset));
+        }
+
+        final boolean addBufferSuc = this.popBufferMergeService.addCk(
+            ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()
+        );
+
+        if (addBufferSuc) {
+            return;
+        }
+
+        this.popBufferMergeService.addCkJustOffset(
+            ck, reviveQid, -1, getMessageTmpResult.getNextBeginOffset()
+        );
+    }
+
+    private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,
+        final int queueId) {
+        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());
+
+        long storeTimestamp = 0;
+        try {
+            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
+            for (ByteBuffer bb : messageBufferList) {
+
+                byteBuffer.put(bb);
+                storeTimestamp = bb.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION);
+            }
+        } finally {
+            getMessageResult.release();
+        }
+
+        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,
+            this.brokerController.getMessageStore().now() - storeTimestamp);
+        return byteBuffer.array();
+    }
+
+    public class PopLongPollingService extends ServiceThread {
+
+        private long lastCleanTime = 0;
+
+        @Override
+        public String getServiceName() {
+            if (PopMessageProcessor.this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+                return PopMessageProcessor.this.brokerController.getBrokerIdentity().getLoggerIdentifier() + PopLongPollingService.class.getSimpleName();
+            }
+            return PopLongPollingService.class.getSimpleName();
+        }
+
+        private void cleanUnusedResource() {
+            try {
+                {
+                    Iterator<Entry<String, ConcurrentHashMap<String, Byte>>> topicCidMapIter = topicCidMap.entrySet().iterator();
+                    while (topicCidMapIter.hasNext()) {
+                        Entry<String, ConcurrentHashMap<String, Byte>> entry = topicCidMapIter.next();
+                        String topic = entry.getKey();
+                        if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
+                            POP_LOGGER.info("remove not exit topic {} in topicCidMap!", topic);
+                            topicCidMapIter.remove();
+                            continue;
+                        }
+                        Iterator<Entry<String, Byte>> cidMapIter = entry.getValue().entrySet().iterator();
+                        while (cidMapIter.hasNext()) {
+                            Entry<String, Byte> cidEntry = cidMapIter.next();
+                            String cid = cidEntry.getKey();
+                            if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
+                                POP_LOGGER.info("remove not exit sub {} of topic {} in topicCidMap!", cid, topic);
+                                cidMapIter.remove();
+                            }
+                        }
+                    }
+                }
+
+                {
+                    Iterator<Entry<String, ConcurrentSkipListSet<PopRequest>>> pollingMapIter = pollingMap.entrySet().iterator();
+                    while (pollingMapIter.hasNext()) {
+                        Entry<String, ConcurrentSkipListSet<PopRequest>> entry = pollingMapIter.next();
+                        if (entry.getKey() == null) {
+                            continue;
+                        }
+                        String[] keyArray = entry.getKey().split(PopAckConstants.SPLIT);
+                        if (keyArray == null || keyArray.length != 3) {
+                            continue;
+                        }
+                        String topic = keyArray[0];
+                        String cid = keyArray[1];
+                        if (brokerController.getTopicConfigManager().selectTopicConfig(topic) == null) {
+                            POP_LOGGER.info("remove not exit topic {} in pollingMap!", topic);
+                            pollingMapIter.remove();
+                            continue;
+                        }
+                        if (!brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().containsKey(cid)) {
+                            POP_LOGGER.info("remove not exit sub {} of topic {} in pollingMap!", cid, topic);
+                            pollingMapIter.remove();
+                            continue;
+                        }
+                    }
+                }
+            } catch (Throwable e) {
+                POP_LOGGER.error("cleanUnusedResource", e);
+            }
+
+            lastCleanTime = System.currentTimeMillis();
+        }
+
+        @Override
+        public void run() {
+            int i = 0;
+            while (!this.stopped) {
+                try {
+                    this.waitForRunning(20);
+                    i++;
+                    if (pollingMap.isEmpty()) {
+                        continue;
+                    }
+                    long tmpTotalPollingNum = 0;
+                    Iterator<Entry<String, ConcurrentSkipListSet<PopRequest>>> pollingMapIterator = pollingMap.entrySet().iterator();
+                    while (pollingMapIterator.hasNext()) {
+                        Entry<String, ConcurrentSkipListSet<PopRequest>> entry = pollingMapIterator.next();
+                        String key = entry.getKey();
+                        ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();
+                        if (popQ == null) {
+                            continue;
+                        }
+                        PopRequest first;
+                        do {
+                            first = popQ.pollFirst();
+                            if (first == null) {
+                                break;
+                            }
+                            if (!first.isTimeout()) {
+                                if (popQ.add(first)) {
+                                    break;
+                                } else {
+                                    POP_LOGGER.info("polling, add fail again: {}", first);
+                                }
+                            }
+                            if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                                POP_LOGGER.info("timeout , wakeUp polling : {}", first);
+                            }
+                            totalPollingNum.decrementAndGet();
+                            wakeUp(first);
+                        }
+                        while (true);
+                        if (i >= 100) {
+                            long tmpPollingNum = popQ.size();
+                            tmpTotalPollingNum = tmpTotalPollingNum + tmpPollingNum;
+                            if (tmpPollingNum > 100) {
+                                POP_LOGGER.info("polling queue {} , size={} ", key, tmpPollingNum);
+                            }
+                        }
+                    }
+
+                    if (i >= 100) {
+                        POP_LOGGER.info("pollingMapSize={},tmpTotalSize={},atomicTotalSize={},diffSize={}",
+                            pollingMap.size(), tmpTotalPollingNum, totalPollingNum.get(),
+                            Math.abs(totalPollingNum.get() - tmpTotalPollingNum));
+                        totalPollingNum.set(tmpTotalPollingNum);
+                        i = 0;
+                    }
+
+                    // clean unused
+                    if (lastCleanTime == 0 || System.currentTimeMillis() - lastCleanTime > 5 * 60 * 1000) {
+                        cleanUnusedResource();
+                    }
+                } catch (Throwable e) {
+                    POP_LOGGER.error("checkPolling error", e);
+                }
+            }
+            // clean all;
+            try {
+                Iterator<Entry<String, ConcurrentSkipListSet<PopRequest>>> pollingMapIterator = pollingMap.entrySet().iterator();
+                while (pollingMapIterator.hasNext()) {
+                    Entry<String, ConcurrentSkipListSet<PopRequest>> entry = pollingMapIterator.next();
+                    ConcurrentSkipListSet<PopRequest> popQ = entry.getValue();
+                    PopRequest first;
+                    while ((first = popQ.pollFirst()) != null) {
+                        wakeUp(first);
+                    }
+                }
+            } catch (Throwable e) {
+            }
+        }
+    }
+
+    static class TimedLock {
+        private final AtomicBoolean lock;
+        private volatile long lockTime;
+
+        public TimedLock() {
+            this.lock = new AtomicBoolean(true);
+            this.lockTime = System.currentTimeMillis();
+        }
+
+        public boolean tryLock() {
+            boolean ret = lock.compareAndSet(true, false);
+            if (ret) {
+                this.lockTime = System.currentTimeMillis();
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+        public void unLock() {
+            lock.set(true);
+        }
+
+        public boolean isLock() {
+            return !lock.get();
+        }
+
+        public long getLockTime() {
+            return lockTime;
+        }
+    }
+
+    public class QueueLockManager extends ServiceThread {
+        private ConcurrentHashMap<String, TimedLock> expiredLocalCache = new ConcurrentHashMap<>(100000);
+
+        public boolean tryLock(String key) {
+            TimedLock timedLock = expiredLocalCache.get(key);
+
+            if (timedLock == null) {
+                TimedLock old = expiredLocalCache.putIfAbsent(key, new TimedLock());
+                if (old != null) {
+                    return false;
+                } else {
+                    timedLock = expiredLocalCache.get(key);
+                }
+            }
+
+            if (timedLock == null) {
+                return false;
+            }
+
+            return timedLock.tryLock();
+        }
+
+        /**
+         * is not thread safe, may cause duplicate lock
+         *
+         * @param usedExpireMillis
+         * @return
+         */
+        public int cleanUnusedLock(final long usedExpireMillis) {
+            Iterator<Entry<String, TimedLock>> iterator = expiredLocalCache.entrySet().iterator();
+
+            int total = 0;
+            while (iterator.hasNext()) {
+                Entry<String, TimedLock> entry = iterator.next();
+
+                if (System.currentTimeMillis() - entry.getValue().getLockTime() > usedExpireMillis) {
+                    iterator.remove();
+                    POP_LOGGER.info("Remove unused queue lock: {}, {}, {}", entry.getKey(),
+                        entry.getValue().getLockTime(),
+                        entry.getValue().isLock());
+                }
+
+                total++;
+            }
+
+            return total;
+        }
+
+        public void unLock(String key) {
+            TimedLock timedLock = expiredLocalCache.get(key);
+            if (timedLock != null) {
+                timedLock.unLock();
+            }
+        }
+
+        @Override
+        public String getServiceName() {
+            if (PopMessageProcessor.this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+                return PopMessageProcessor.this.brokerController.getBrokerIdentity().getLoggerIdentifier() + QueueLockManager.class.getSimpleName();
+            }
+            return QueueLockManager.class.getSimpleName();
+        }
+
+        @Override
+        public void run() {
+            while (!isStopped()) {
+                try {
+                    this.waitForRunning(60000);
+                    int count = cleanUnusedLock(60000);
+                    POP_LOGGER.info("QueueLockSize={}", count);
+                } catch (Exception e) {
+                    PopMessageProcessor.POP_LOGGER.error("QueueLockManager run error", e);
+                }
+            }
+        }
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java

@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.alibaba.fastjson.JSON;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.util.MsgUtil;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.utils.DataConverter;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.pop.AckMsg;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+
+public class PopReviveService extends ServiceThread {
+    private static final InternalLogger POP_LOGGER = InternalLoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
+
+    private int queueId;
+    private BrokerController brokerController;
+    private String reviveTopic;
+    private volatile boolean shouldRunPopRevive = false;
+
+    public PopReviveService(BrokerController brokerController, String reviveTopic, int queueId) {
+        this.queueId = queueId;
+        this.brokerController = brokerController;
+        this.reviveTopic = reviveTopic;
+    }
+
+    @Override
+    public String getServiceName() {
+        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return brokerController.getBrokerIdentity().getLoggerIdentifier() + "PopReviveService_" + this.queueId;
+        }
+        return "PopReviveService_" + this.queueId;
+    }
+
+    public void setShouldRunPopRevive(final boolean shouldRunPopRevive) {
+        this.shouldRunPopRevive = shouldRunPopRevive;
+    }
+
+    public boolean isShouldRunPopRevive() {
+        return shouldRunPopRevive;
+    }
+
+    private void reviveRetry(PopCheckPoint popCheckPoint, MessageExt messageExt) throws Exception {
+        if (!shouldRunPopRevive) {
+            POP_LOGGER.info("slave skip retry , revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+            return;
+        }
+        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
+        if (!popCheckPoint.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+            msgInner.setTopic(KeyBuilder.buildPopRetryTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()));
+        } else {
+            msgInner.setTopic(popCheckPoint.getTopic());
+        }
+        msgInner.setBody(messageExt.getBody());
+        msgInner.setQueueId(0);
+        if (messageExt.getTags() != null) {
+            msgInner.setTags(messageExt.getTags());
+        } else {
+            MessageAccessor.setProperties(msgInner, new HashMap<String, String>());
+        }
+        msgInner.setBornTimestamp(messageExt.getBornTimestamp());
+        msgInner.setBornHost(brokerController.getStoreHost());
+        msgInner.setStoreHost(brokerController.getStoreHost());
+        msgInner.setReconsumeTimes(messageExt.getReconsumeTimes() + 1);
+        msgInner.getProperties().putAll(messageExt.getProperties());
+        if (messageExt.getReconsumeTimes() == 0 || msgInner.getProperties().get(MessageConst.PROPERTY_FIRST_POP_TIME) == null) {
+            msgInner.getProperties().put(MessageConst.PROPERTY_FIRST_POP_TIME, String.valueOf(popCheckPoint.getPopTime()));
+        }
+        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
+        addRetryTopicIfNoExit(msgInner.getTopic(), popCheckPoint.getCId());
+        PutMessageResult putMessageResult = brokerController.getMessageStore().putMessage(msgInner);
+        if (brokerController.getBrokerConfig().isEnablePopLog()) {
+            POP_LOGGER.info("reviveQueueId={},retry msg , ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ",
+                queueId, popCheckPoint, messageExt.getQueueId(), messageExt.getQueueOffset(),
+                (System.currentTimeMillis() - popCheckPoint.getReviveTime()) / 1000, putMessageResult);
+        }
+        if (putMessageResult.getAppendMessageResult() == null || putMessageResult.getAppendMessageResult().getStatus() != AppendMessageStatus.PUT_OK) {
+            throw new Exception("reviveQueueId=" + queueId + ",revive error ,msg is :" + msgInner);
+        }
+        this.brokerController.getBrokerStatsManager().incBrokerPutNums(1);
+        this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());
+        this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes());
+        if (brokerController.getPopMessageProcessor() != null) {
+            brokerController.getPopMessageProcessor().notifyMessageArriving(
+                KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()),
+                popCheckPoint.getCId(),
+                -1
+            );
+        }
+    }
+
+    private void initPopRetryOffset(String topic, String consumerGroup) {
+        long offset = this.brokerController.getConsumerOffsetManager().queryOffset(consumerGroup, topic, 0);
+        if (offset < 0) {
+            this.brokerController.getConsumerOffsetManager().commitOffset("initPopRetryOffset", consumerGroup, topic,
+                0, 0);
+        }
+    }
+
+    private void addRetryTopicIfNoExit(String topic, String consumerGroup) {
+        if (brokerController != null) {
+            TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic);
+            if (topicConfig != null) {
+                return;
+            }
+            topicConfig = new TopicConfig(topic);
+            topicConfig.setReadQueueNums(PopAckConstants.retryQueueNum);
+            topicConfig.setWriteQueueNums(PopAckConstants.retryQueueNum);
+            topicConfig.setTopicFilterType(TopicFilterType.SINGLE_TAG);
+            topicConfig.setPerm(6);
+            topicConfig.setTopicSysFlag(0);
+            brokerController.getTopicConfigManager().updateTopicConfig(topicConfig);
+
+            initPopRetryOffset(topic, consumerGroup);
+        }
+    }
+
+    private List<MessageExt> getReviveMessage(long offset, int queueId) {
+        PullResult pullResult = getMessage(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, offset, 32);
+        if (pullResult == null) {
+            return null;
+        }
+        if (reachTail(pullResult, offset)) {
+            POP_LOGGER.info("reviveQueueId={}, reach tail,offset {}", queueId, offset);
+        } else if (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL || pullResult.getPullStatus() == PullStatus.NO_MATCHED_MSG) {
+            POP_LOGGER.error("reviveQueueId={}, OFFSET_ILLEGAL {}, result is {}", queueId, offset, pullResult);
+            if (!shouldRunPopRevive) {
+                POP_LOGGER.info("slave skip offset correct topic={}, reviveQueueId={}", reviveTopic, queueId);
+                return null;
+            }
+            this.brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, pullResult.getNextBeginOffset() - 1);
+        }
+        return pullResult.getMsgFoundList();
+    }
+
+    private boolean reachTail(PullResult pullResult, long offset) {
+        return pullResult.getPullStatus() == PullStatus.NO_NEW_MSG
+            || (pullResult.getPullStatus() == PullStatus.OFFSET_ILLEGAL && offset == pullResult.getMaxOffset());
+    }
+
+    private MessageExt getBizMessage(String topic, long offset, int queueId, String brokerName) {
+        return this.brokerController.getEscapeBridge().getMessage(topic, offset, queueId, brokerName);
+    }
+
+    public PullResult getMessage(String group, String topic, int queueId, long offset, int nums) {
+        GetMessageResult getMessageResult = this.brokerController.getMessageStore().getMessage(group, topic, queueId, offset, nums, null);
+
+        if (getMessageResult != null) {
+            PullStatus pullStatus = PullStatus.NO_NEW_MSG;
+            List<MessageExt> foundList = null;
+            switch (getMessageResult.getStatus()) {
+                case FOUND:
+                    pullStatus = PullStatus.FOUND;
+                    foundList = decodeMsgList(getMessageResult);
+                    brokerController.getBrokerStatsManager().incGroupGetNums(group, topic, getMessageResult.getMessageCount());
+                    brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize());
+                    brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
+                    brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId,
+                        brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1).getStoreTimestamp());
+                    break;
+                case NO_MATCHED_MESSAGE:
+                    pullStatus = PullStatus.NO_MATCHED_MSG;
+                    POP_LOGGER.warn("no matched message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
+                        getMessageResult.getStatus(), topic, group, offset);
+                    break;
+                case NO_MESSAGE_IN_QUEUE:
+                    pullStatus = PullStatus.NO_NEW_MSG;
+                    POP_LOGGER.warn("no new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
+                        getMessageResult.getStatus(), topic, group, offset);
+                    break;
+                case MESSAGE_WAS_REMOVING:
+                case NO_MATCHED_LOGIC_QUEUE:
+                case OFFSET_FOUND_NULL:
+                case OFFSET_OVERFLOW_BADLY:
+                case OFFSET_OVERFLOW_ONE:
+                case OFFSET_TOO_SMALL:
+                    pullStatus = PullStatus.OFFSET_ILLEGAL;
+                    POP_LOGGER.warn("offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}",
+                        getMessageResult.getStatus(), topic, group, offset);
+                    break;
+                default:
+                    assert false;
+                    break;
+            }
+
+            return new PullResult(pullStatus, getMessageResult.getNextBeginOffset(), getMessageResult.getMinOffset(),
+                getMessageResult.getMaxOffset(), foundList);
+
+        } else {
+            POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}", topic, group, offset);
+            return null;
+        }
+    }
+
+    private List<MessageExt> decodeMsgList(GetMessageResult getMessageResult) {
+        List<MessageExt> foundList = new ArrayList<>();
+        try {
+            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
+            if (messageBufferList != null) {
+                for (int i = 0; i < messageBufferList.size(); i++) {
+                    ByteBuffer bb = messageBufferList.get(i);
+                    if (bb == null) {
+                        POP_LOGGER.error("bb is null {}", getMessageResult);
+                        continue;
+                    }
+                    MessageExt msgExt = MessageDecoder.decode(bb);
+                    if (msgExt == null) {
+                        POP_LOGGER.error("decode msgExt is null {}", getMessageResult);
+                        continue;
+                    }
+                    // use CQ offset, not offset in Message
+                    msgExt.setQueueOffset(getMessageResult.getMessageQueueOffset().get(i));
+                    foundList.add(msgExt);
+                }
+            }
+        } finally {
+            getMessageResult.release();
+        }
+
+        return foundList;
+    }
+
+    private void consumeReviveMessage(ConsumeReviveObj consumeReviveObj) {
+        HashMap<String, PopCheckPoint> map = consumeReviveObj.map;
+        long startScanTime = System.currentTimeMillis();
+        long endTime = 0;
+        long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(PopAckConstants.REVIVE_GROUP, reviveTopic, queueId);
+        consumeReviveObj.oldOffset = oldOffset;
+        POP_LOGGER.info("reviveQueueId={}, old offset is {} ", queueId, oldOffset);
+        long offset = oldOffset + 1;
+        int noMsgCount = 0;
+        long firstRt = 0;
+        // offset self amend
+        while (true) {
+            if (!shouldRunPopRevive) {
+                POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                break;
+            }
+            List<MessageExt> messageExts = getReviveMessage(offset, queueId);
+            if (messageExts == null || messageExts.isEmpty()) {
+                long old = endTime;
+                // move endTime
+                if (endTime != 0 && ((System.currentTimeMillis() - endTime) > (3 * PopAckConstants.SECOND))) {
+                    endTime = System.currentTimeMillis();
+                }
+                POP_LOGGER.info("reviveQueueId={}, offset is {}, can not get new msg, old endTime {}, new endTime {}",
+                    queueId, offset, old, endTime);
+                if (endTime - firstRt > PopAckConstants.ackTimeInterval + PopAckConstants.SECOND) {
+                    break;
+                }
+                noMsgCount++;
+                try {
+                    Thread.sleep(100);
+                } catch (Throwable e) {
+                }
+                if (noMsgCount * 100 > 4 * PopAckConstants.SECOND) {
+                    break;
+                } else {
+                    continue;
+                }
+            } else {
+                noMsgCount = 0;
+            }
+            if (System.currentTimeMillis() - startScanTime > brokerController.getBrokerConfig().getReviveScanTime()) {
+                POP_LOGGER.info("reviveQueueId={}, scan timeout  ", queueId);
+                break;
+            }
+            for (MessageExt messageExt : messageExts) {
+                if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) {
+                    String raw = new String(messageExt.getBody(), DataConverter.charset);
+                    if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                        POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
+                    }
+                    PopCheckPoint point = JSON.parseObject(raw, PopCheckPoint.class);
+                    if (point.getTopic() == null || point.getCId() == null) {
+                        continue;
+                    }
+                    map.put(point.getTopic() + point.getCId() + point.getQueueId() + point.getStartOffset() + point.getPopTime(), point);
+                    point.setReviveOffset(messageExt.getQueueOffset());
+                    if (firstRt == 0) {
+                        firstRt = point.getReviveTime();
+                    }
+                } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {
+                    String raw = new String(messageExt.getBody(), DataConverter.charset);
+                    if (brokerController.getBrokerConfig().isEnablePopLog()) {
+                        POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
+                    }
+                    AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class);
+                    PopCheckPoint point = map.get(ackMsg.getTopic() + ackMsg.getConsumerGroup() + ackMsg.getQueueId() + ackMsg.getStartOffset() + ackMsg.getPopTime());
+                    if (point == null) {
+                        continue;
+                    }
+                    int indexOfAck = point.indexOfAck(ackMsg.getAckOffset());
+                    if (indexOfAck > -1) {
+                        point.setBitMap(DataConverter.setBit(point.getBitMap(), indexOfAck, true));
+                    } else {
+                        POP_LOGGER.error("invalid ack index, {}, {}", ackMsg, point);
+                    }
+                }
+                long deliverTime = MsgUtil.getMessageDeliverTime(this.brokerController, messageExt);
+                if (deliverTime > endTime) {
+                    endTime = deliverTime;
+                }
+            }
+            offset = offset + messageExts.size();
+        }
+        consumeReviveObj.endTime = endTime;
+    }
+
+    private void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwable {
+        ArrayList<PopCheckPoint> sortList = consumeReviveObj.genSortList();
+        POP_LOGGER.info("reviveQueueId={},ck listSize={}", queueId, sortList.size());
+        if (sortList.size() != 0) {
+            POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={} ; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(),
+                sortList.get(0).getReviveOffset(), sortList.get(sortList.size() - 1).getStartOffset(), sortList.get(sortList.size() - 1).getReviveOffset());
+        }
+        long newOffset = consumeReviveObj.oldOffset;
+        for (PopCheckPoint popCheckPoint : sortList) {
+            if (!shouldRunPopRevive) {
+                POP_LOGGER.info("slave skip ck process , revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                break;
+            }
+            if (consumeReviveObj.endTime - popCheckPoint.getReviveTime() <= (PopAckConstants.ackTimeInterval + PopAckConstants.SECOND)) {
+                break;
+            }
+
+            // check normal topic, skip ck , if normal topic is not exist
+            String normalTopic = KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId());
+            if (brokerController.getTopicConfigManager().selectTopicConfig(normalTopic) == null) {
+                POP_LOGGER.warn("reviveQueueId={},can not get normal topic {} , then continue ", queueId, popCheckPoint.getTopic());
+                newOffset = popCheckPoint.getReviveOffset();
+                continue;
+            }
+            if (null == brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(popCheckPoint.getCId())) {
+                POP_LOGGER.warn("reviveQueueId={},can not get cid {} , then continue ", queueId, popCheckPoint.getCId());
+                newOffset = popCheckPoint.getReviveOffset();
+                continue;
+            }
+
+            reviveMsgFromCk(popCheckPoint);
+
+            newOffset = popCheckPoint.getReviveOffset();
+        }
+        if (newOffset > consumeReviveObj.oldOffset) {
+            if (!shouldRunPopRevive) {
+                POP_LOGGER.info("slave skip commit, revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                return;
+            }
+            this.brokerController.getConsumerOffsetManager().commitOffset(PopAckConstants.LOCAL_HOST, PopAckConstants.REVIVE_GROUP, reviveTopic, queueId, newOffset);
+        }
+        consumeReviveObj.newOffset = newOffset;
+    }
+
+    private void reviveMsgFromCk(PopCheckPoint popCheckPoint) throws Throwable {
+        for (int j = 0; j < popCheckPoint.getNum(); j++) {
+            if (DataConverter.getBit(popCheckPoint.getBitMap(), j)) {
+                continue;
+            }
+
+            // retry msg
+            long msgOffset = popCheckPoint.ackOffsetByIndex((byte) j);
+            MessageExt messageExt = getBizMessage(popCheckPoint.getTopic(), msgOffset, popCheckPoint.getQueueId(), popCheckPoint.getBrokerName());
+            if (messageExt == null) {
+                POP_LOGGER.warn("reviveQueueId={},can not get biz msg topic is {}, offset is {} , then continue ",
+                    queueId, popCheckPoint.getTopic(), msgOffset);
+                continue;
+            }
+            //skip ck from last epoch
+            if (popCheckPoint.getPopTime() < messageExt.getStoreTimestamp()) {
+                POP_LOGGER.warn("reviveQueueId={},skip ck from last epoch {}", queueId, popCheckPoint);
+                continue;
+            }
+            reviveRetry(popCheckPoint, messageExt);
+        }
+    }
+
+    @Override
+    public void run() {
+        int slow = 1;
+        while (!this.isStopped()) {
+            try {
+                if (System.currentTimeMillis() < brokerController.getShouldStartTime()) {
+                    POP_LOGGER.info("PopReviveService Ready to run after {}", brokerController.getShouldStartTime());
+                    this.waitForRunning(1000);
+                    continue;
+                }
+                this.waitForRunning(brokerController.getBrokerConfig().getReviveInterval());
+                if (!shouldRunPopRevive) {
+                    POP_LOGGER.info("skip start revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                    continue;
+                }
+
+                POP_LOGGER.info("start revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                ConsumeReviveObj consumeReviveObj = new ConsumeReviveObj();
+                consumeReviveMessage(consumeReviveObj);
+
+                if (!shouldRunPopRevive) {
+                    POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId);
+                    continue;
+                }
+
+                mergeAndRevive(consumeReviveObj);
+
+                ArrayList<PopCheckPoint> sortList = consumeReviveObj.sortList;
+                long delay = 0;
+                if (sortList != null && !sortList.isEmpty()) {
+                    delay = (System.currentTimeMillis() - sortList.get(0).getReviveTime()) / 1000;
+                    slow = 1;
+                }
+
+                POP_LOGGER.info("reviveQueueId={},revive finish,old offset is {}, new offset is {}, ckDelay={}  ",
+                    queueId, consumeReviveObj.oldOffset, consumeReviveObj.newOffset, delay);
+
+                if (sortList == null || sortList.isEmpty()) {
+                    POP_LOGGER.info("reviveQueueId={},has no new msg ,take a rest {}", queueId, slow);
+                    this.waitForRunning(slow * brokerController.getBrokerConfig().getReviveInterval());
+                    if (slow < brokerController.getBrokerConfig().getReviveMaxSlow()) {
+                        slow++;
+                    }
+                }
+
+            } catch (Throwable e) {
+                POP_LOGGER.error("reviveQueueId=" + queueId + ",revive error", e);
+            }
+        }
+    }
+
+    static class ConsumeReviveObj {
+        HashMap<String, PopCheckPoint> map = new HashMap<>();
+        ArrayList<PopCheckPoint> sortList;
+        long oldOffset;
+        long endTime;
+        long newOffset;
+
+        ArrayList<PopCheckPoint> genSortList() {
+            if (sortList != null) {
+                return sortList;
+            }
+            sortList = new ArrayList<>(map.values());
+            Collections.sort(sortList, new Comparator<PopCheckPoint>() {
+                @Override
+                public int compare(PopCheckPoint o1, PopCheckPoint o2) {
+                    return (int) (o1.getReviveOffset() - o2.getReviveOffset());
+                }
+            });
+            return sortList;
+        }
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java

@@ -16,65 +16,263 @@
  */
 package org.apache.rocketmq.broker.processor;
 
+import java.util.List;
+
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.ChannelFutureListener;
 import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.FileRegion;
-import java.nio.ByteBuffer;
-import java.util.List;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
 import org.apache.rocketmq.broker.filter.ConsumerFilterData;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter;
 import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
-import org.apache.rocketmq.broker.longpolling.PullRequest;
+import org.apache.rocketmq.broker.mqtrace.AbortProcessException;
 import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
 import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
-import org.apache.rocketmq.broker.pagecache.ManyMessageTransfer;
+import org.apache.rocketmq.broker.plugin.PullMessageResultHandler;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.filter.ExpressionType;
 import org.apache.rocketmq.common.filter.FilterAPI;
 import org.apache.rocketmq.common.help.FAQUrl;
-import org.apache.rocketmq.common.message.MessageDecoder;
-import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.ForbiddenType;
+import org.apache.rocketmq.common.protocol.NamespaceUtil;
+import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
 import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
 import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent;
+import org.apache.rocketmq.common.rpc.RpcClientUtils;
+import org.apache.rocketmq.common.rpc.RpcRequest;
+import org.apache.rocketmq.common.rpc.RpcResponse;
+import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
 import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.sysflag.PullSysFlag;
-import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
-import org.apache.rocketmq.remoting.common.RemotingUtil;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.netty.RequestTask;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.MessageFilter;
-import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
-public class PullMessageProcessor extends AsyncNettyRequestProcessor {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-    private final BrokerController brokerController;
+import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
+
+public class PullMessageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private List<ConsumeMessageHook> consumeMessageHookList;
+    private PullMessageResultHandler pullMessageResultHandler;
+    private final BrokerController brokerController;
 
     public PullMessageProcessor(final BrokerController brokerController) {
         this.brokerController = brokerController;
+        this.pullMessageResultHandler = new DefaultPullMessageResultHandler(brokerController);
+    }
+
+    private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader,
+        TopicQueueMappingContext mappingContext) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+            String topic = mappingContext.getTopic();
+            Integer globalId = mappingContext.getGlobalId();
+            // if the leader? consider the order consumer, which will lock the mq
+            if (!mappingContext.isLeader()) {
+                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d cannot find mapping item in request process of current broker %s", topic, globalId, mappingDetail.getBname()));
+            }
+
+            Long globalOffset = requestHeader.getQueueOffset();
+            LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);
+            mappingContext.setCurrentItem(mappingItem);
+
+            if (globalOffset < mappingItem.getLogicOffset()) {
+                //handleOffsetMoved
+                //If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently
+                //Otherwise, we could just transfer it to the physical process
+            }
+            //below are physical info
+            String bname = mappingItem.getBname();
+            Integer phyQueueId = mappingItem.getQueueId();
+            Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset);
+            requestHeader.setQueueId(phyQueueId);
+            requestHeader.setQueueOffset(phyQueueOffset);
+            if (mappingItem.checkIfEndOffsetDecided()
+                && requestHeader.getMaxMsgNums() != null) {
+                requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums()));
+            }
+
+            if (mappingDetail.getBname().equals(bname)) {
+                //just let it go, do the local pull process
+                return null;
+            }
+
+            int sysFlag = requestHeader.getSysFlag();
+            requestHeader.setLo(false);
+            requestHeader.setBname(bname);
+            sysFlag = PullSysFlag.clearSuspendFlag(sysFlag);
+            sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);
+            requestHeader.setSysFlag(sysFlag);
+            RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null);
+            RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
+            if (rpcResponse.getException() != null) {
+                throw rpcResponse.getException();
+            }
+
+            PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader();
+            {
+                RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode());
+                if (rewriteResult != null) {
+                    return rewriteResult;
+                }
+            }
+            return RpcClientUtils.createCommandForRpcResponse(rpcResponse);
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
+    }
+
+    private RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader,
+        PullMessageResponseHeader responseHeader,
+        TopicQueueMappingContext mappingContext, final int code) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+            LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem();
+
+            LogicQueueMappingItem currentItem = mappingContext.getCurrentItem();
+
+            LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);
+
+            assert currentItem.getLogicOffset() >= 0;
+
+            long requestOffset = requestHeader.getQueueOffset();
+            long nextBeginOffset = responseHeader.getNextBeginOffset();
+            long minOffset = responseHeader.getMinOffset();
+            long maxOffset = responseHeader.getMaxOffset();
+            int responseCode = code;
+
+            //consider the following situations
+            // 1. read from slave, currently not supported
+            // 2. the middle queue is truncated because of deleting commitlog
+            if (code != ResponseCode.SUCCESS) {
+                //note the currentItem maybe both the leader and  the earliest
+                boolean isRevised = false;
+                if (leaderItem.getGen() == currentItem.getGen()) {
+                    //read the leader
+                    if (requestOffset > maxOffset) {
+                        //actually, we need do nothing, but keep the code structure here
+                        if (code == ResponseCode.PULL_OFFSET_MOVED) {
+                            responseCode = ResponseCode.PULL_OFFSET_MOVED;
+                            nextBeginOffset = maxOffset;
+                        } else {
+                            //maybe current broker is the slave
+                            responseCode = code;
+                        }
+                    } else if (requestOffset < minOffset) {
+                        nextBeginOffset = minOffset;
+                        responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+                    } else {
+                        responseCode = code;
+                    }
+                }
+                //note the currentItem maybe both the leader and  the earliest
+                if (earlistItem.getGen() == currentItem.getGen()) {
+                    //read the earliest one
+                    if (requestOffset < minOffset) {
+                        if (code == ResponseCode.PULL_OFFSET_MOVED) {
+                            responseCode = ResponseCode.PULL_OFFSET_MOVED;
+                            nextBeginOffset = minOffset;
+                        } else {
+                            //maybe read from slave, but we still set it to moved
+                            responseCode = ResponseCode.PULL_OFFSET_MOVED;
+                            nextBeginOffset = minOffset;
+                        }
+                    } else if (requestOffset >= maxOffset) {
+                        //just move to another item
+                        LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+                        if (nextItem != null) {
+                            isRevised = true;
+                            currentItem = nextItem;
+                            nextBeginOffset = currentItem.getStartOffset();
+                            minOffset = currentItem.getStartOffset();
+                            maxOffset = minOffset;
+                            responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+                        } else {
+                            //maybe the next one's logic offset is -1
+                            responseCode = ResponseCode.PULL_NOT_FOUND;
+                        }
+                    } else {
+                        //let it go
+                        responseCode = code;
+                    }
+                }
+
+                //read from the middle item, ignore the PULL_OFFSET_MOVED
+                if (!isRevised
+                    && leaderItem.getGen() != currentItem.getGen()
+                    && earlistItem.getGen() != currentItem.getGen()) {
+                    if (requestOffset < minOffset) {
+                        nextBeginOffset = minOffset;
+                        responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+                    } else if (requestOffset >= maxOffset) {
+                        //just move to another item
+                        LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
+                        if (nextItem != null) {
+                            currentItem = nextItem;
+                            nextBeginOffset = currentItem.getStartOffset();
+                            minOffset = currentItem.getStartOffset();
+                            maxOffset = minOffset;
+                            responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
+                        } else {
+                            //maybe the next one's logic offset is -1
+                            responseCode = ResponseCode.PULL_NOT_FOUND;
+                        }
+                    } else {
+                        responseCode = code;
+                    }
+                }
+            }
+
+            //handle nextBeginOffset
+            //the next begin offset should no more than the end offset
+            if (currentItem.checkIfEndOffsetDecided()
+                && nextBeginOffset >= currentItem.getEndOffset()) {
+                nextBeginOffset = currentItem.getEndOffset();
+            }
+            responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset));
+            //handle min offset
+            responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset)));
+            //handle max offset
+            responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset),
+                TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId())));
+            //set the offsetDelta
+            responseHeader.setOffsetDelta(currentItem.computeOffsetDelta());
+
+            if (code != ResponseCode.SUCCESS) {
+                return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader);
+            } else {
+                return null;
+            }
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
     }
 
     @Override
@@ -85,12 +283,15 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
     @Override
     public boolean rejectRequest() {
+        if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()
+            && this.brokerController.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
+            return true;
+        }
         return false;
     }
 
     private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
         throws RemotingCommandException {
-        final long beginTimeMills = this.brokerController.getMessageStore().now();
         RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
         final PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) response.readCustomHeader();
         final PullMessageRequestHeader requestHeader =
@@ -98,7 +299,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
         response.setOpaque(request.getOpaque());
 
-        log.debug("receive PullMessage request command, {}", request);
+        LOGGER.debug("receive PullMessage request command, {}", request);
 
         if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
             response.setCode(ResponseCode.NO_PERMISSION);
@@ -106,6 +307,13 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
             return response;
         }
 
+        if (request.getCode() == RequestCode.LITE_PULL_MESSAGE && !this.brokerController.getBrokerConfig().isLitePullMessageEnable()) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark(
+                "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] for lite pull consumer is forbidden");
+            return response;
+        }
+
         SubscriptionGroupConfig subscriptionGroupConfig =
             this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getConsumerGroup());
         if (null == subscriptionGroupConfig) {
@@ -116,6 +324,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
         if (!subscriptionGroupConfig.isConsumeEnable()) {
             response.setCode(ResponseCode.NO_PERMISSION);
+            responseHeader.setForbiddenType(ForbiddenType.GROUP_FORBIDDEN);
             response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup());
             return response;
         }
@@ -128,7 +337,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
         if (null == topicConfig) {
-            log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
+            LOGGER.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
             response.setCode(ResponseCode.TOPIC_NOT_EXIST);
             response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
             return response;
@@ -136,14 +345,24 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
         if (!PermName.isReadable(topicConfig.getPerm())) {
             response.setCode(ResponseCode.NO_PERMISSION);
+            responseHeader.setForbiddenType(ForbiddenType.TOPIC_FORBIDDEN);
             response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden");
             return response;
         }
 
+        TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false);
+
+        {
+            RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
+            if (rewriteResult != null) {
+                return rewriteResult;
+            }
+        }
+
         if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
             String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
                 requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
-            log.warn(errorInfo);
+            LOGGER.warn(errorInfo);
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.setRemark(errorInfo);
             return response;
@@ -164,7 +383,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                     assert consumerFilterData != null;
                 }
             } catch (Exception e) {
-                log.warn("Parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(),
+                LOGGER.warn("Parse the consumer's subscription[{}] failed, group: {}", requestHeader.getSubscription(),
                     requestHeader.getConsumerGroup());
                 response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED);
                 response.setRemark("parse the consumer's subscription failed");
@@ -174,7 +393,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
             ConsumerGroupInfo consumerGroupInfo =
                 this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
             if (null == consumerGroupInfo) {
-                log.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
+                LOGGER.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
                 response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
                 response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
                 return response;
@@ -183,20 +402,30 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
             if (!subscriptionGroupConfig.isConsumeBroadcastEnable()
                 && consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) {
                 response.setCode(ResponseCode.NO_PERMISSION);
+                responseHeader.setForbiddenType(ForbiddenType.BROADCASTING_DISABLE_FORBIDDEN);
                 response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] can not consume by broadcast way");
                 return response;
             }
 
+            boolean readForbidden = this.brokerController.getSubscriptionGroupManager().getForbidden(//
+                subscriptionGroupConfig.getGroupName(), requestHeader.getTopic(), PermName.INDEX_PERM_READ);
+            if (readForbidden) {
+                response.setCode(ResponseCode.NO_PERMISSION);
+                responseHeader.setForbiddenType(ForbiddenType.SUBSCRIPTION_FORBIDDEN);
+                response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] is forbidden for topic[" + requestHeader.getTopic() + "]");
+                return response;
+            }
+
             subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic());
             if (null == subscriptionData) {
-                log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic());
+                LOGGER.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic());
                 response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
                 response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
                 return response;
             }
 
             if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) {
-                log.warn("The broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(),
+                LOGGER.warn("The broker's subscription is not latest, group: {} {}", requestHeader.getConsumerGroup(),
                     subscriptionData.getSubString());
                 response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST);
                 response.setRemark("the consumer's subscription not latest");
@@ -211,7 +440,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                     return response;
                 }
                 if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {
-                    log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
+                    LOGGER.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
                         requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
                     response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);
                     response.setRemark("the consumer's consumer filter data not latest");
@@ -243,38 +472,10 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
             response.setRemark(getMessageResult.getStatus().name());
             responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());
             responseHeader.setMinOffset(getMessageResult.getMinOffset());
+            // this does not need to be modified since it's not an accurate value under logical queue.
             responseHeader.setMaxOffset(getMessageResult.getMaxOffset());
-
-            if (getMessageResult.isSuggestPullingFromSlave()) {
-                responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
-            } else {
-                responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
-            }
-
-            switch (this.brokerController.getMessageStoreConfig().getBrokerRole()) {
-                case ASYNC_MASTER:
-                case SYNC_MASTER:
-                    break;
-                case SLAVE:
-                    if (!this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
-                        response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
-                        responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
-                    }
-                    break;
-            }
-
-            if (this.brokerController.getBrokerConfig().isSlaveReadEnable()) {
-                // consume too slow ,redirect to another machine
-                if (getMessageResult.isSuggestPullingFromSlave()) {
-                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
-                }
-                // consume ok
-                else {
-                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
-                }
-            } else {
-                responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
-            }
+            responseHeader.setTopicSysFlag(topicConfig.getTopicSysFlag());
+            responseHeader.setGroupSysFlag(subscriptionGroupConfig.getGroupSysFlag());
 
             switch (getMessageResult.getStatus()) {
                 case FOUND:
@@ -287,9 +488,8 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                 case NO_MESSAGE_IN_QUEUE:
                     if (0 != requestHeader.getQueueOffset()) {
                         response.setCode(ResponseCode.PULL_OFFSET_MOVED);
-
                         // XXX: warn and notify me
-                        log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",
+                        LOGGER.info("the broker stores no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",
                             requestHeader.getQueueOffset(),
                             getMessageResult.getNextBeginOffset(),
                             requestHeader.getTopic(),
@@ -309,15 +509,15 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                 case OFFSET_OVERFLOW_BADLY:
                     response.setCode(ResponseCode.PULL_OFFSET_MOVED);
                     // XXX: warn and notify me
-                    log.info("the request offset: {} over flow badly, broker max offset: {}, consumer: {}",
-                        requestHeader.getQueueOffset(), getMessageResult.getMaxOffset(), channel.remoteAddress());
+                    LOGGER.info("the request offset: {} over flow badly, fix to {}, broker max offset: {}, consumer: {}",
+                        requestHeader.getQueueOffset(), getMessageResult.getNextBeginOffset(), getMessageResult.getMaxOffset(), channel.remoteAddress());
                     break;
                 case OFFSET_OVERFLOW_ONE:
                     response.setCode(ResponseCode.PULL_NOT_FOUND);
                     break;
                 case OFFSET_TOO_SMALL:
                     response.setCode(ResponseCode.PULL_OFFSET_MOVED);
-                    log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}",
+                    LOGGER.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}",
                         requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(),
                         getMessageResult.getMinOffset(), channel.remoteAddress());
                     break;
@@ -326,13 +526,50 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                     break;
             }
 
+            if (this.brokerController.getBrokerConfig().isSlaveReadEnable() && !this.brokerController.getBrokerConfig().isInBrokerContainer()) {
+                // consume too slow ,redirect to another machine
+                if (getMessageResult.isSuggestPullingFromSlave()) {
+                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getWhichBrokerWhenConsumeSlowly());
+                }
+                // consume ok
+                else {
+                    responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
+                }
+            } else {
+                responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
+            }
+
+            if (this.brokerController.getBrokerConfig().getBrokerId() != MixAll.MASTER_ID && !getMessageResult.isSuggestPullingFromSlave()) {
+                if (this.brokerController.getMinBrokerIdInGroup() == MixAll.MASTER_ID) {
+                    LOGGER.debug("slave redirect pullRequest to master, topic: {}, queueId: {}, consumer group: {}, next: {}, min: {}, max: {}",
+                        requestHeader.getTopic(),
+                        requestHeader.getQueueId(),
+                        requestHeader.getConsumerGroup(),
+                        responseHeader.getNextBeginOffset(),
+                        responseHeader.getMinOffset(),
+                        responseHeader.getMaxOffset()
+                    );
+                    responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
+                    if (!getMessageResult.getStatus().equals(GetMessageStatus.FOUND)) {
+                        response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
+                    }
+                }
+            }
+
             if (this.hasConsumeMessageHook()) {
+                String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+                String authType = request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE);
+                String ownerParent = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT);
+                String ownerSelf = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF);
+
                 ConsumeMessageContext context = new ConsumeMessageContext();
                 context.setConsumerGroup(requestHeader.getConsumerGroup());
                 context.setTopic(requestHeader.getTopic());
                 context.setQueueId(requestHeader.getQueueId());
-
-                String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+                context.setAccountAuthType(authType);
+                context.setAccountOwnerParent(ownerParent);
+                context.setAccountOwnerSelf(ownerSelf);
+                context.setNamespace(NamespaceUtil.getNamespaceFromResource(requestHeader.getTopic()));
 
                 switch (response.getCode()) {
                     case ResponseCode.SUCCESS:
@@ -344,6 +581,11 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                         context.setCommercialRcvSize(getMessageResult.getBufferTotalSize());
                         context.setCommercialOwner(owner);
 
+                        context.setRcvStat(BrokerStatsManager.StatsType.RCV_SUCCESS);
+                        context.setRcvMsgNum(getMessageResult.getMessageCount());
+                        context.setRcvMsgSize(getMessageResult.getBufferTotalSize());
+                        context.setCommercialRcvMsgNum(getMessageResult.getMsgCount4Commercial());
+
                         break;
                     case ResponseCode.PULL_NOT_FOUND:
                         if (!brokerAllowSuspend) {
@@ -352,6 +594,10 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                             context.setCommercialRcvTimes(1);
                             context.setCommercialOwner(owner);
 
+                            context.setRcvStat(BrokerStatsManager.StatsType.RCV_EPOLLS);
+                            context.setRcvMsgNum(0);
+                            context.setRcvMsgSize(0);
+                            context.setCommercialRcvMsgNum(0);
                         }
                         break;
                     case ResponseCode.PULL_RETRY_IMMEDIATELY:
@@ -359,102 +605,43 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                         context.setCommercialRcvStats(BrokerStatsManager.StatsType.RCV_EPOLLS);
                         context.setCommercialRcvTimes(1);
                         context.setCommercialOwner(owner);
+
+                        context.setRcvStat(BrokerStatsManager.StatsType.RCV_EPOLLS);
+                        context.setRcvMsgNum(0);
+                        context.setRcvMsgSize(0);
+                        context.setCommercialRcvMsgNum(0);
                         break;
                     default:
                         assert false;
                         break;
                 }
 
-                this.executeConsumeMessageHookBefore(context);
+                try {
+                    this.executeConsumeMessageHookBefore(context);
+                } catch (AbortProcessException e) {
+                    response.setCode(e.getResponseCode());
+                    response.setRemark(e.getErrorMessage());
+                    return response;
+                }
             }
 
-            switch (response.getCode()) {
-                case ResponseCode.SUCCESS:
-
-                    this.brokerController.getBrokerStatsManager().incGroupGetNums(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
-                        getMessageResult.getMessageCount());
-
-                    this.brokerController.getBrokerStatsManager().incGroupGetSize(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
-                        getMessageResult.getBufferTotalSize());
-
-                    this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount());
-                    if (this.brokerController.getBrokerConfig().isTransferMsgByHeap()) {
-                        final byte[] r = this.readGetMessageResult(getMessageResult, requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
-                        this.brokerController.getBrokerStatsManager().incGroupGetLatency(requestHeader.getConsumerGroup(),
-                            requestHeader.getTopic(), requestHeader.getQueueId(),
-                            (int) (this.brokerController.getMessageStore().now() - beginTimeMills));
-                        response.setBody(r);
-                    } else {
-                        try {
-                            FileRegion fileRegion =
-                                new ManyMessageTransfer(response.encodeHeader(getMessageResult.getBufferTotalSize()), getMessageResult);
-                            channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() {
-                                @Override
-                                public void operationComplete(ChannelFuture future) throws Exception {
-                                    getMessageResult.release();
-                                    if (!future.isSuccess()) {
-                                        log.error("transfer many message by pagecache failed, {}", channel.remoteAddress(), future.cause());
-                                    }
-                                }
-                            });
-                        } catch (Throwable e) {
-                            log.error("transfer many message by pagecache exception", e);
-                            getMessageResult.release();
-                        }
-
-                        response = null;
-                    }
-                    break;
-                case ResponseCode.PULL_NOT_FOUND:
-
-                    if (brokerAllowSuspend && hasSuspendFlag) {
-                        long pollingTimeMills = suspendTimeoutMillisLong;
-                        if (!this.brokerController.getBrokerConfig().isLongPollingEnable()) {
-                            pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
-                        }
-
-                        String topic = requestHeader.getTopic();
-                        long offset = requestHeader.getQueueOffset();
-                        int queueId = requestHeader.getQueueId();
-                        PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
-                            this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
-                        this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
-                        response = null;
-                        break;
-                    }
-
-                case ResponseCode.PULL_RETRY_IMMEDIATELY:
-                    break;
-                case ResponseCode.PULL_OFFSET_MOVED:
-                    if (this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE
-                        || this.brokerController.getMessageStoreConfig().isOffsetCheckInSlave()) {
-                        MessageQueue mq = new MessageQueue();
-                        mq.setTopic(requestHeader.getTopic());
-                        mq.setQueueId(requestHeader.getQueueId());
-                        mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
-
-                        OffsetMovedEvent event = new OffsetMovedEvent();
-                        event.setConsumerGroup(requestHeader.getConsumerGroup());
-                        event.setMessageQueue(mq);
-                        event.setOffsetRequest(requestHeader.getQueueOffset());
-                        event.setOffsetNew(getMessageResult.getNextBeginOffset());
-                        this.generateOffsetMovedEvent(event);
-                        log.warn(
-                            "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}",
-                            requestHeader.getTopic(), requestHeader.getConsumerGroup(), event.getOffsetRequest(), event.getOffsetNew(),
-                            responseHeader.getSuggestWhichBrokerId());
-                    } else {
-                        responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId());
-                        response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY);
-                        log.warn("PULL_OFFSET_MOVED:none correction. topic={}, groupId={}, requestOffset={}, suggestBrokerId={}",
-                            requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getQueueOffset(),
-                            responseHeader.getSuggestWhichBrokerId());
-                    }
-
-                    break;
-                default:
-                    assert false;
+            //rewrite the response for the
+            RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());
+            if (rewriteResult != null) {
+                response = rewriteResult;
             }
+
+            response = this.pullMessageResultHandler.handle(
+                getMessageResult,
+                request,
+                requestHeader,
+                channel,
+                subscriptionData,
+                subscriptionGroupConfig,
+                brokerAllowSuspend,
+                messageFilter,
+                response
+            );
         } else {
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.setRemark("store getMessage return null");
@@ -462,8 +649,6 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
 
         boolean storeOffsetEnable = brokerAllowSuspend;
         storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag;
-        storeOffsetEnable = storeOffsetEnable
-            && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;
         if (storeOffsetEnable) {
             this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel),
                 requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
@@ -486,66 +671,6 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
         }
     }
 
-    private byte[] readGetMessageResult(final GetMessageResult getMessageResult, final String group, final String topic,
-        final int queueId) {
-        final ByteBuffer byteBuffer = ByteBuffer.allocate(getMessageResult.getBufferTotalSize());
-
-        long storeTimestamp = 0;
-        try {
-            List<ByteBuffer> messageBufferList = getMessageResult.getMessageBufferList();
-            for (ByteBuffer bb : messageBufferList) {
-
-                byteBuffer.put(bb);
-                int sysFlag = bb.getInt(MessageDecoder.SYSFLAG_POSITION);
-//                bornhost has the IPv4 ip if the MessageSysFlag.BORNHOST_V6_FLAG bit of sysFlag is 0
-//                IPv4 host = ip(4 byte) + port(4 byte); IPv6 host = ip(16 byte) + port(4 byte)
-                int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20;
-                int msgStoreTimePos = 4 // 1 TOTALSIZE
-                    + 4 // 2 MAGICCODE
-                    + 4 // 3 BODYCRC
-                    + 4 // 4 QUEUEID
-                    + 4 // 5 FLAG
-                    + 8 // 6 QUEUEOFFSET
-                    + 8 // 7 PHYSICALOFFSET
-                    + 4 // 8 SYSFLAG
-                    + 8 // 9 BORNTIMESTAMP
-                    + bornhostLength; // 10 BORNHOST
-                storeTimestamp = bb.getLong(msgStoreTimePos);
-            }
-        } finally {
-            getMessageResult.release();
-        }
-
-        this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - storeTimestamp);
-        return byteBuffer.array();
-    }
-
-    private void generateOffsetMovedEvent(final OffsetMovedEvent event) {
-        try {
-            MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
-            msgInner.setTopic(TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT);
-            msgInner.setTags(event.getConsumerGroup());
-            msgInner.setDelayTimeLevel(0);
-            msgInner.setKeys(event.getConsumerGroup());
-            msgInner.setBody(event.encode());
-            msgInner.setFlag(0);
-            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-            msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(TopicFilterType.SINGLE_TAG, msgInner.getTags()));
-
-            msgInner.setQueueId(0);
-            msgInner.setSysFlag(0);
-            msgInner.setBornTimestamp(System.currentTimeMillis());
-            msgInner.setBornHost(RemotingUtil.string2SocketAddress(this.brokerController.getBrokerAddr()));
-            msgInner.setStoreHost(msgInner.getBornHost());
-
-            msgInner.setReconsumeTimes(0);
-
-            PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
-        } catch (Exception e) {
-            log.warn(String.format("generateOffsetMovedEvent Exception, %s", event.toString()), e);
-        }
-    }
-
     public void executeRequestWhenWakeup(final Channel channel,
         final RemotingCommand request) throws RemotingCommandException {
         Runnable run = new Runnable() {
@@ -562,21 +687,21 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
                                 @Override
                                 public void operationComplete(ChannelFuture future) throws Exception {
                                     if (!future.isSuccess()) {
-                                        log.error("processRequestWrapper response to {} failed",
+                                        LOGGER.error("processRequestWrapper response to {} failed",
                                             future.channel().remoteAddress(), future.cause());
-                                        log.error(request.toString());
-                                        log.error(response.toString());
+                                        LOGGER.error(request.toString());
+                                        LOGGER.error(response.toString());
                                     }
                                 }
                             });
                         } catch (Throwable e) {
-                            log.error("processRequestWrapper process request over, but response failed", e);
-                            log.error(request.toString());
-                            log.error(response.toString());
+                            LOGGER.error("processRequestWrapper process request over, but response failed", e);
+                            LOGGER.error(request.toString());
+                            LOGGER.error(response.toString());
                         }
                     }
                 } catch (RemotingCommandException e1) {
-                    log.error("excuteRequestWhenWakeup run", e1);
+                    LOGGER.error("excuteRequestWhenWakeup run", e1);
                 }
             }
         };
@@ -586,4 +711,8 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor {
     public void registerConsumeMessageHook(List<ConsumeMessageHook> consumeMessageHookList) {
         this.consumeMessageHookList = consumeMessageHookList;
     }
+
+    public void setPullMessageResultHandler(PullMessageResultHandler pullMessageResultHandler) {
+        this.pullMessageResultHandler = pullMessageResultHandler;
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessor.java

@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.ChannelHandlerContext;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
+import org.apache.rocketmq.broker.loadbalance.AssignmentManager;
+import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;
+import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.message.MessageQueueAssignment;
+import org.apache.rocketmq.common.message.MessageRequestMode;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody;
+import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+public class QueryAssignmentProcessor implements NettyRequestProcessor {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    private final BrokerController brokerController;
+
+    private final ConcurrentHashMap<String, AllocateMessageQueueStrategy> name2LoadStrategy = new ConcurrentHashMap<String, AllocateMessageQueueStrategy>();
+
+    private MessageRequestModeManager messageRequestModeManager;
+
+    public QueryAssignmentProcessor(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+
+        //register strategy
+        //NOTE: init with broker's log instead of init with ClientLogger.getLog();
+        AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(log);
+        name2LoadStrategy.put(allocateMessageQueueAveragely.getName(), allocateMessageQueueAveragely);
+        AllocateMessageQueueAveragelyByCircle allocateMessageQueueAveragelyByCircle = new AllocateMessageQueueAveragelyByCircle(log);
+        name2LoadStrategy.put(allocateMessageQueueAveragelyByCircle.getName(), allocateMessageQueueAveragelyByCircle);
+
+        this.messageRequestModeManager = new MessageRequestModeManager(brokerController);
+        this.messageRequestModeManager.load();
+    }
+
+    @Override
+    public RemotingCommand processRequest(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        switch (request.getCode()) {
+            case RequestCode.QUERY_ASSIGNMENT:
+                return this.queryAssignment(ctx, request);
+            case RequestCode.SET_MESSAGE_REQUEST_MODE:
+                return this.setMessageRequestMode(ctx, request);
+            default:
+                break;
+        }
+        return null;
+    }
+
+    @Override
+    public boolean rejectRequest() {
+        return false;
+    }
+
+    /**
+     *
+     */
+    private RemotingCommand queryAssignment(ChannelHandlerContext ctx, RemotingCommand request)
+        throws RemotingCommandException {
+        final QueryAssignmentRequestBody requestBody = QueryAssignmentRequestBody.decode(request.getBody(), QueryAssignmentRequestBody.class);
+        final String topic = requestBody.getTopic();
+        final String consumerGroup = requestBody.getConsumerGroup();
+        final String clientId = requestBody.getClientId();
+        final MessageModel messageModel = requestBody.getMessageModel();
+        final String strategyName = requestBody.getStrategyName();
+
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        final QueryAssignmentResponseBody responseBody = new QueryAssignmentResponseBody();
+
+        SetMessageRequestModeRequestBody setMessageRequestModeRequestBody = this.messageRequestModeManager.getMessageRequestMode(topic, consumerGroup);
+
+        if (setMessageRequestModeRequestBody == null) {
+            setMessageRequestModeRequestBody = new SetMessageRequestModeRequestBody();
+            setMessageRequestModeRequestBody.setTopic(topic);
+            setMessageRequestModeRequestBody.setConsumerGroup(consumerGroup);
+
+            if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+                // retry topic must be pull mode
+                setMessageRequestModeRequestBody.setMode(MessageRequestMode.PULL);
+            } else {
+                setMessageRequestModeRequestBody.setMode(brokerController.getBrokerConfig().getDefaultMessageRequestMode());
+            }
+
+            if (setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) {
+                setMessageRequestModeRequestBody.setPopShareQueueNum(brokerController.getBrokerConfig().getDefaultPopShareQueueNum());
+            }
+        }
+
+        Set<MessageQueue> messageQueues = doLoadBalance(topic, consumerGroup, clientId, messageModel, strategyName, setMessageRequestModeRequestBody, ctx);
+
+        Set<MessageQueueAssignment> assignments = null;
+        if (messageQueues != null) {
+            assignments = new HashSet<MessageQueueAssignment>();
+            for (MessageQueue messageQueue : messageQueues) {
+                MessageQueueAssignment messageQueueAssignment = new MessageQueueAssignment();
+                messageQueueAssignment.setMessageQueue(messageQueue);
+                if (setMessageRequestModeRequestBody != null) {
+                    messageQueueAssignment.setMode(setMessageRequestModeRequestBody.getMode());
+                }
+                assignments.add(messageQueueAssignment);
+            }
+        }
+
+        responseBody.setMessageQueueAssignments(assignments);
+        response.setBody(responseBody.encode());
+        response.setCode(ResponseCode.SUCCESS);
+        response.setRemark(null);
+        return response;
+    }
+
+    /**
+     * Returns empty set means the client should clear all load assigned to it before, null means invalid result and the
+     * client should skip the update logic
+     *
+     * @param topic
+     * @param consumerGroup
+     * @param clientId
+     * @param messageModel
+     * @param strategyName
+     * @return the MessageQueues assigned to this client
+     */
+    private Set<MessageQueue> doLoadBalance(final String topic, final String consumerGroup, final String clientId,
+        final MessageModel messageModel, final String strategyName,
+        SetMessageRequestModeRequestBody setMessageRequestModeRequestBody, final ChannelHandlerContext ctx) {
+        Set<MessageQueue> assignedQueueSet = null;
+        AssignmentManager assignmentManager = brokerController.getAssignmentManager();
+
+        switch (messageModel) {
+            case BROADCASTING: {
+                assignedQueueSet = assignmentManager.getTopicSubscribeInfo(topic);
+                if (assignedQueueSet == null) {
+                    log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic);
+                }
+                break;
+            }
+            case CLUSTERING: {
+                Set<MessageQueue> mqSet = assignmentManager.getTopicSubscribeInfo(topic);
+                if (null == mqSet) {
+                    if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+                        log.warn("QueryLoad: no assignment for group[{}], the topic[{}] does not exist.", consumerGroup, topic);
+                    }
+                    return null;
+                }
+
+                if (!brokerController.getBrokerConfig().isServerLoadBalancerEnable()) {
+                    return mqSet;
+                }
+
+                List<String> cidAll = null;
+                ConsumerGroupInfo consumerGroupInfo = this.brokerController.getConsumerManager().getConsumerGroupInfo(consumerGroup);
+                if (consumerGroupInfo != null) {
+                    cidAll = consumerGroupInfo.getAllClientId();
+                }
+                if (null == cidAll) {
+                    log.warn("QueryLoad: no assignment for group[{}] topic[{}], get consumer id list failed", consumerGroup, topic);
+                    return null;
+                }
+
+                List<MessageQueue> mqAll = new ArrayList<MessageQueue>();
+                mqAll.addAll(mqSet);
+                Collections.sort(mqAll);
+                Collections.sort(cidAll);
+                List<MessageQueue> allocateResult = null;
+
+                try {
+                    AllocateMessageQueueStrategy allocateMessageQueueStrategy = name2LoadStrategy.get(strategyName);
+                    if (null == allocateMessageQueueStrategy) {
+                        log.warn("QueryLoad: unsupported strategy [{}],  {}", strategyName, RemotingHelper.parseChannelRemoteAddr(ctx.channel()));
+                        return null;
+                    }
+
+                    if (setMessageRequestModeRequestBody != null && setMessageRequestModeRequestBody.getMode() == MessageRequestMode.POP) {
+                        allocateResult = allocate4Pop(allocateMessageQueueStrategy, consumerGroup, clientId, mqAll,
+                            cidAll, setMessageRequestModeRequestBody.getPopShareQueueNum());
+
+                    } else {
+                        allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll);
+                    }
+                } catch (Throwable e) {
+                    log.error("QueryLoad: no assignment for group[{}] topic[{}], allocate message queue exception. strategy name: {}, ex: {}", consumerGroup, topic, strategyName, e);
+                    return null;
+                }
+
+                assignedQueueSet = new HashSet<MessageQueue>();
+                if (allocateResult != null) {
+                    assignedQueueSet.addAll(allocateResult);
+                }
+                break;
+            }
+            default:
+                break;
+        }
+        return assignedQueueSet;
+    }
+
+    public List<MessageQueue> allocate4Pop(AllocateMessageQueueStrategy allocateMessageQueueStrategy,
+        final String consumerGroup, final String clientId, List<MessageQueue> mqAll, List<String> cidAll,
+        int popShareQueueNum) {
+
+        List<MessageQueue> allocateResult;
+        if (popShareQueueNum <= 0 || popShareQueueNum >= cidAll.size() - 1) {
+            //each client pop all messagequeue
+            allocateResult = new ArrayList<>(mqAll.size());
+            for (MessageQueue mq : mqAll) {
+                //must create new MessageQueue in case of change cache in AssignmentManager
+                MessageQueue newMq = new MessageQueue(mq.getTopic(), mq.getBrokerName(), -1);
+                allocateResult.add(newMq);
+            }
+
+        } else {
+            if (cidAll.size() <= mqAll.size()) {
+                //consumer working in pop mode could share the MessageQueues assigned to the N (N = popWorkGroupSize) consumer following it in the cid list
+                allocateResult = allocateMessageQueueStrategy.allocate(consumerGroup, clientId, mqAll, cidAll);
+                int index = cidAll.indexOf(clientId);
+                if (index >= 0) {
+                    for (int i = 1; i <= popShareQueueNum; i++) {
+                        index++;
+                        index = index % cidAll.size();
+                        List<MessageQueue> tmp = allocateMessageQueueStrategy.allocate(consumerGroup, cidAll.get(index), mqAll, cidAll);
+                        allocateResult.addAll(tmp);
+                    }
+                }
+            } else {
+                //make sure each cid is assigned
+                allocateResult = allocate(consumerGroup, clientId, mqAll, cidAll);
+            }
+        }
+
+        return allocateResult;
+    }
+
+    private List<MessageQueue> allocate(String consumerGroup, String currentCID, List<MessageQueue> mqAll,
+        List<String> cidAll) {
+        if (StringUtils.isBlank(currentCID)) {
+            throw new IllegalArgumentException("currentCID is empty");
+        }
+
+        if (CollectionUtils.isEmpty(mqAll)) {
+            throw new IllegalArgumentException("mqAll is null or mqAll empty");
+        }
+        if (CollectionUtils.isEmpty(cidAll)) {
+            throw new IllegalArgumentException("cidAll is null or cidAll empty");
+        }
+
+        List<MessageQueue> result = new ArrayList<MessageQueue>();
+        if (!cidAll.contains(currentCID)) {
+            log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}",
+                consumerGroup,
+                currentCID,
+                cidAll);
+            return result;
+        }
+
+        int index = cidAll.indexOf(currentCID);
+        result.add(mqAll.get(index % mqAll.size()));
+        return result;
+    }
+
+    private RemotingCommand setMessageRequestMode(ChannelHandlerContext ctx,
+        RemotingCommand request) throws RemotingCommandException {
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        final SetMessageRequestModeRequestBody requestBody = SetMessageRequestModeRequestBody.decode(request.getBody(), SetMessageRequestModeRequestBody.class);
+
+        final String topic = requestBody.getTopic();
+        if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+            response.setCode(ResponseCode.NO_PERMISSION);
+            response.setRemark("retry topic is not allowed to set mode");
+            return response;
+        }
+
+        final String consumerGroup = requestBody.getConsumerGroup();
+
+        this.messageRequestModeManager.setMessageRequestMode(topic, consumerGroup, requestBody);
+        this.messageRequestModeManager.persist();
+
+        response.setCode(ResponseCode.SUCCESS);
+        response.setRemark(null);
+        return response;
+    }
+
+    public MessageRequestModeManager getMessageRequestModeManager() {
+        return messageRequestModeManager;
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/processor/QueryMessageProcessor.java

@@ -33,14 +33,13 @@ import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader;
 import org.apache.rocketmq.common.protocol.header.ViewMessageRequestHeader;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.QueryMessageResult;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 
-public class QueryMessageProcessor extends AsyncNettyRequestProcessor {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
-
+public class QueryMessageProcessor implements NettyRequestProcessor {
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
 
     public QueryMessageProcessor(final BrokerController brokerController) {
@@ -106,12 +105,12 @@ public class QueryMessageProcessor extends AsyncNettyRequestProcessor {
                     public void operationComplete(ChannelFuture future) throws Exception {
                         queryMessageResult.release();
                         if (!future.isSuccess()) {
-                            log.error("transfer query message by page cache failed, ", future.cause());
+                            LOGGER.error("transfer query message by page cache failed, ", future.cause());
                         }
                     }
                 });
             } catch (Throwable e) {
-                log.error("", e);
+                LOGGER.error("", e);
                 queryMessageResult.release();
             }
 
@@ -146,12 +145,12 @@ public class QueryMessageProcessor extends AsyncNettyRequestProcessor {
                     public void operationComplete(ChannelFuture future) throws Exception {
                         selectMappedBufferResult.release();
                         if (!future.isSuccess()) {
-                            log.error("Transfer one message from page cache failed, ", future.cause());
+                            LOGGER.error("Transfer one message from page cache failed, ", future.cause());
                         }
                     }
                 });
             } catch (Throwable e) {
-                log.error("", e);
+                LOGGER.error("", e);
                 selectMappedBufferResult.release();
             }
 

broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java

@@ -40,7 +40,7 @@ import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
@@ -116,7 +116,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
         }
 
         response.setCode(-1);
-        super.msgCheck(ctx, requestHeader, response);
+        super.msgCheck(ctx, requestHeader, request, response);
         if (response.getCode() != -1) {
             return response;
         }
@@ -254,7 +254,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
                 break;
 
             // Failed
-            case CREATE_MAPEDFILE_FAILED:
+            case CREATE_MAPPED_FILE_FAILED:
                 log.warn("create mapped file failed, server is busy or broken.");
                 break;
             case MESSAGE_ILLEGAL:
@@ -271,7 +271,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
                     "service not available now. It may be caused by one of the following reasons: " +
                         "the broker's disk is full, messages are put to the slave, message store has been shut down, etc.");
                 break;
-            case OS_PAGECACHE_BUSY:
+            case OS_PAGE_CACHE_BUSY:
                 log.warn("[PC_SYNCHRONIZED]broker busy, start flow control for a while");
                 break;
             case UNKNOWN_ERROR:
@@ -283,6 +283,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
         }
 
         String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+        int commercialSizePerMsg = brokerController.getBrokerConfig().getCommercialSizePerMsg();
         if (putOk) {
             this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1);
             this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(),
@@ -300,7 +301,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
 
                 int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();
                 int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes();
-                int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount;
+                int incValue = (int) Math.ceil(wroteSize / commercialSizePerMsg) * commercialBaseCount;
 
                 sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS);
                 sendMessageContext.setCommercialSendTimes(incValue);
@@ -310,7 +311,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor {
         } else {
             if (hasSendMessageHook()) {
                 int wroteSize = request.getBody().length;
-                int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT);
+                int incValue = (int) Math.ceil(wroteSize / commercialSizePerMsg);
 
                 sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE);
                 sendMessageContext.setCommercialSendTimes(incValue);

broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageCallback.java

@@ -15,15 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.remoting.netty;
+package org.apache.rocketmq.broker.processor;
 
-import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
-public abstract class AsyncNettyRequestProcessor implements NettyRequestProcessor {
-
-    public void asyncProcessRequest(ChannelHandlerContext ctx, RemotingCommand request, RemotingResponseCallback responseCallback) throws Exception {
-        RemotingCommand response = processRequest(ctx, request);
-        responseCallback.callback(response);
-    }
+public interface SendMessageCallback {
+    /**
+     * On send complete.
+     *
+     * @param ctx send context
+     * @param response send response
+     */
+    void onComplete(SendMessageContext ctx, RemotingCommand response);
 }

broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java

@@ -16,54 +16,54 @@
  */
 package org.apache.rocketmq.broker.processor;
 
-import java.net.SocketAddress;
-import java.util.List;
+import java.nio.ByteBuffer;
 import java.util.Map;
+import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ThreadLocalRandom;
 
 import io.netty.channel.ChannelHandlerContext;
 import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
-import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
+import org.apache.rocketmq.broker.mqtrace.AbortProcessException;
 import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
+import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
 import org.apache.rocketmq.common.MQVersion;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.help.FAQUrl;
 import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageClientIDSetter;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.message.MessageExtBatch;
-import org.apache.rocketmq.common.protocol.NamespaceUtil;
 import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
-import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
 import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
-import org.apache.rocketmq.common.sysflag.TopicSysFlag;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.QueueTypeUtils;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
-import org.apache.rocketmq.remoting.netty.RemotingResponseCallback;
+import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
-import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
-public class SendMessageProcessor extends AbstractSendMessageProcessor {
+import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
 
-    private List<ConsumeMessageHook> consumeMessageHookList;
+public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor {
 
     public SendMessageProcessor(final BrokerController brokerController) {
         super(brokerController);
@@ -71,279 +71,94 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
 
     @Override
     public RemotingCommand processRequest(ChannelHandlerContext ctx,
-                                          RemotingCommand request) throws RemotingCommandException {
-        RemotingCommand response = null;
-        try {
-            response = asyncProcessRequest(ctx, request).get();
-        } catch (InterruptedException | ExecutionException e) {
-            log.error("process SendMessage error, request : " + request.toString(), e);
-        }
-        return response;
-    }
-
-    @Override
-    public void asyncProcessRequest(ChannelHandlerContext ctx, RemotingCommand request, RemotingResponseCallback responseCallback) throws Exception {
-        asyncProcessRequest(ctx, request).thenAcceptAsync(responseCallback::callback, this.brokerController.getPutMessageFutureExecutor());
-    }
-
-    public CompletableFuture<RemotingCommand> asyncProcessRequest(ChannelHandlerContext ctx,
-                                                                  RemotingCommand request) throws RemotingCommandException {
-        final SendMessageContext mqtraceContext;
+        RemotingCommand request) throws RemotingCommandException {
+        SendMessageContext traceContext;
         switch (request.getCode()) {
             case RequestCode.CONSUMER_SEND_MSG_BACK:
-                return this.asyncConsumerSendMsgBack(ctx, request);
+                return this.consumerSendMsgBack(ctx, request);
             default:
                 SendMessageRequestHeader requestHeader = parseRequestHeader(request);
                 if (requestHeader == null) {
-                    return CompletableFuture.completedFuture(null);
+                    return null;
                 }
-                mqtraceContext = buildMsgContext(ctx, requestHeader);
-                this.executeSendMessageHookBefore(ctx, request, mqtraceContext);
+                TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true);
+                RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext);
+                if (rewriteResult != null) {
+                    return rewriteResult;
+                }
+                traceContext = buildMsgContext(ctx, requestHeader);
+                String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+                traceContext.setCommercialOwner(owner);
+                try {
+                    this.executeSendMessageHookBefore(ctx, request, traceContext);
+                } catch (AbortProcessException e) {
+                    final RemotingCommand errorResponse = RemotingCommand.createResponseCommand(e.getResponseCode(), e.getErrorMessage());
+                    errorResponse.setOpaque(request.getOpaque());
+                    return errorResponse;
+                }
+
+                RemotingCommand response;
                 if (requestHeader.isBatch()) {
-                    return this.asyncSendBatchMessage(ctx, request, mqtraceContext, requestHeader);
+                    response = this.sendBatchMessage(ctx, request, traceContext, requestHeader, mappingContext,
+                        (ctx1, response1) -> executeSendMessageHookAfter(response1, ctx1));
                 } else {
-                    return this.asyncSendMessage(ctx, request, mqtraceContext, requestHeader);
+                    response = this.sendMessage(ctx, request, traceContext, requestHeader, mappingContext,
+                        (ctx12, response12) -> executeSendMessageHookAfter(response12, ctx12));
                 }
+
+                return response;
         }
     }
 
     @Override
     public boolean rejectRequest() {
-        return this.brokerController.getMessageStore().isOSPageCacheBusy() ||
-            this.brokerController.getMessageStore().isTransientStorePoolDeficient();
+        if (!this.brokerController.getBrokerConfig().isEnableSlaveActingMaster() && this.brokerController.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
+            return true;
+        }
+
+        if (this.brokerController.getMessageStore().isOSPageCacheBusy() || this.brokerController.getMessageStore().isTransientStorePoolDeficient()) {
+            return true;
+        }
+
+        return false;
     }
 
-    private CompletableFuture<RemotingCommand> asyncConsumerSendMsgBack(ChannelHandlerContext ctx,
-                                                                        RemotingCommand request) throws RemotingCommandException {
-        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
-        final ConsumerSendMsgBackRequestHeader requestHeader =
-                (ConsumerSendMsgBackRequestHeader)request.decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class);
-        String namespace = NamespaceUtil.getNamespaceFromResource(requestHeader.getGroup());
-        if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) {
-            ConsumeMessageContext context = buildConsumeMessageContext(namespace, requestHeader, request);
-            this.executeConsumeMessageHookAfter(context);
-        }
-        SubscriptionGroupConfig subscriptionGroupConfig =
-            this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(requestHeader.getGroup());
-        if (null == subscriptionGroupConfig) {
-            response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST);
-            response.setRemark("subscription group not exist, " + requestHeader.getGroup() + " "
-                + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST));
-            return CompletableFuture.completedFuture(response);
-        }
-        if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())) {
-            response.setCode(ResponseCode.NO_PERMISSION);
-            response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending message is forbidden");
-            return CompletableFuture.completedFuture(response);
-        }
+    /**
+     * If the response is not null, it meets some errors
+     *
+     * @return
+     */
 
-        if (subscriptionGroupConfig.getRetryQueueNums() <= 0) {
-            response.setCode(ResponseCode.SUCCESS);
-            response.setRemark(null);
-            return CompletableFuture.completedFuture(response);
-        }
-
-        String newTopic = MixAll.getRetryTopic(requestHeader.getGroup());
-        int queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % subscriptionGroupConfig.getRetryQueueNums();
-        int topicSysFlag = 0;
-        if (requestHeader.isUnitMode()) {
-            topicSysFlag = TopicSysFlag.buildSysFlag(false, true);
-        }
-
-        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(
-            newTopic,
-            subscriptionGroupConfig.getRetryQueueNums(),
-            PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag);
-        if (null == topicConfig) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark("topic[" + newTopic + "] not exist");
-            return CompletableFuture.completedFuture(response);
-        }
-
-        if (!PermName.isWriteable(topicConfig.getPerm())) {
-            response.setCode(ResponseCode.NO_PERMISSION);
-            response.setRemark(String.format("the topic[%s] sending message is forbidden", newTopic));
-            return CompletableFuture.completedFuture(response);
-        }
-        MessageExt msgExt = this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getOffset());
-        if (null == msgExt) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark("look message by offset failed, " + requestHeader.getOffset());
-            return CompletableFuture.completedFuture(response);
-        }
-
-        final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
-        if (null == retryTopic) {
-            MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic());
-        }
-        msgExt.setWaitStoreMsgOK(false);
-
-        int delayLevel = requestHeader.getDelayLevel();
-
-        int maxReconsumeTimes = subscriptionGroupConfig.getRetryMaxTimes();
-        if (request.getVersion() >= MQVersion.Version.V3_4_9.ordinal()) {
-            Integer times = requestHeader.getMaxReconsumeTimes();
-            if (times != null) {
-                maxReconsumeTimes = times;
+    private RemotingCommand rewriteResponseForStaticTopic(SendMessageResponseHeader responseHeader,
+        TopicQueueMappingContext mappingContext) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
             }
-        }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
 
-        if (msgExt.getReconsumeTimes() >= maxReconsumeTimes
-            || delayLevel < 0) {
-            newTopic = MixAll.getDLQTopic(requestHeader.getGroup());
-            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % DLQ_NUMS_PER_GROUP;
-
-            topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,
-                    DLQ_NUMS_PER_GROUP,
-                    PermName.PERM_WRITE | PermName.PERM_READ, 0);
-
-            if (null == topicConfig) {
-                response.setCode(ResponseCode.SYSTEM_ERROR);
-                response.setRemark("topic[" + newTopic + "] not exist");
-                return CompletableFuture.completedFuture(response);
+            LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();
+            if (mappingItem == null) {
+                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));
             }
-            msgExt.setDelayTimeLevel(0);
-        } else {
-            if (0 == delayLevel) {
-                delayLevel = 3 + msgExt.getReconsumeTimes();
+            //no need to care the broker name
+            long staticLogicOffset = mappingItem.computeStaticQueueOffsetLoosely(responseHeader.getQueueOffset());
+            if (staticLogicOffset < 0) {
+                //if the logic offset is -1, just let it go
+                //maybe we need a dynamic config
+                //return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d convert offset error in current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));
             }
-            msgExt.setDelayTimeLevel(delayLevel);
+            responseHeader.setQueueId(mappingContext.getGlobalId());
+            responseHeader.setQueueOffset(staticLogicOffset);
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
         }
-
-        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
-        msgInner.setTopic(newTopic);
-        msgInner.setBody(msgExt.getBody());
-        msgInner.setFlag(msgExt.getFlag());
-        MessageAccessor.setProperties(msgInner, msgExt.getProperties());
-        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
-        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags()));
-
-        msgInner.setQueueId(queueIdInt);
-        msgInner.setSysFlag(msgExt.getSysFlag());
-        msgInner.setBornTimestamp(msgExt.getBornTimestamp());
-        msgInner.setBornHost(msgExt.getBornHost());
-        msgInner.setStoreHost(msgExt.getStoreHost());
-        msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1);
-
-        String originMsgId = MessageAccessor.getOriginMessageId(msgExt);
-        MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId);
-        msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties()));
-
-        CompletableFuture<PutMessageResult> putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner);
-        return putMessageResult.thenApply(r -> {
-            if (r != null) {
-                switch (r.getPutMessageStatus()) {
-                    case PUT_OK:
-                        String backTopic = msgExt.getTopic();
-                        String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
-                        if (correctTopic != null) {
-                            backTopic = correctTopic;
-                        }
-                        if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msgInner.getTopic())) {
-                            this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic());
-                            this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), r.getAppendMessageResult().getWroteBytes());
-                            this.brokerController.getBrokerStatsManager().incQueuePutNums(msgInner.getTopic(), msgInner.getQueueId());
-                            this.brokerController.getBrokerStatsManager().incQueuePutSize(msgInner.getTopic(), msgInner.getQueueId(), r.getAppendMessageResult().getWroteBytes());
-                        }
-                        this.brokerController.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), backTopic);
-                        response.setCode(ResponseCode.SUCCESS);
-                        response.setRemark(null);
-                        return response;
-                    default:
-                        break;
-                }
-                response.setCode(ResponseCode.SYSTEM_ERROR);
-                response.setRemark(r.getPutMessageStatus().name());
-                return response;
-            }
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark("putMessageResult is null");
-            return response;
-        });
-    }
-
-
-    private CompletableFuture<RemotingCommand> asyncSendMessage(ChannelHandlerContext ctx, RemotingCommand request,
-                                                                SendMessageContext mqtraceContext,
-                                                                SendMessageRequestHeader requestHeader) {
-        final RemotingCommand response = preSend(ctx, request, requestHeader);
-        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
-
-        if (response.getCode() != -1) {
-            return CompletableFuture.completedFuture(response);
-        }
-
-        final byte[] body = request.getBody();
-
-        int queueIdInt = requestHeader.getQueueId();
-        TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
-
-        if (queueIdInt < 0) {
-            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());
-        }
-
-        MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
-        msgInner.setTopic(requestHeader.getTopic());
-        msgInner.setQueueId(queueIdInt);
-
-        if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig)) {
-            return CompletableFuture.completedFuture(response);
-        }
-
-        msgInner.setBody(body);
-        msgInner.setFlag(requestHeader.getFlag());
-        Map<String, String> origProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
-        MessageAccessor.setProperties(msgInner, origProps);
-        msgInner.setBornTimestamp(requestHeader.getBornTimestamp());
-        msgInner.setBornHost(ctx.channel().remoteAddress());
-        msgInner.setStoreHost(this.getStoreHost());
-        msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
-        String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
-        MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
-        if (origProps.containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) {
-            // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message.
-            // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it.
-            String waitStoreMsgOKValue = origProps.remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK);
-            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-            // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later
-            origProps.put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue);
-        } else {
-            msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        }
-
-        CompletableFuture<PutMessageResult> putMessageResult = null;
-        String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
-        if (Boolean.parseBoolean(transFlag)) {
-            if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
-                response.setCode(ResponseCode.NO_PERMISSION);
-                response.setRemark(
-                        "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1()
-                                + "] sending transaction message is forbidden");
-                return CompletableFuture.completedFuture(response);
-            }
-            putMessageResult = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner);
-        } else {
-            putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner);
-        }
-        return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt);
-    }
-
-    private CompletableFuture<RemotingCommand> handlePutMessageResultFuture(CompletableFuture<PutMessageResult> putMessageResult,
-                                                                            RemotingCommand response,
-                                                                            RemotingCommand request,
-                                                                            MessageExt msgInner,
-                                                                            SendMessageResponseHeader responseHeader,
-                                                                            SendMessageContext sendMessageContext,
-                                                                            ChannelHandlerContext ctx,
-                                                                            int queueIdInt) {
-        return putMessageResult.thenApply(r ->
-            handlePutMessageResult(r, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt)
-        );
+        return null;
     }
 
     private boolean handleRetryAndDLQ(SendMessageRequestHeader requestHeader, RemotingCommand response,
-                                      RemotingCommand request,
-                                      MessageExt msg, TopicConfig topicConfig) {
+        RemotingCommand request,
+        MessageExt msg, TopicConfig topicConfig, Map<String, String> properties) {
         String newTopic = requestHeader.getTopic();
         if (null != newTopic && newTopic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
             String groupName = newTopic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length());
@@ -361,9 +176,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
                 maxReconsumeTimes = requestHeader.getMaxReconsumeTimes();
             }
             int reconsumeTimes = requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes();
-            if (reconsumeTimes >= maxReconsumeTimes) {
+            // Using '>' instead of '>=' to compatible with the case that reconsumeTimes here are increased by client.
+            if (reconsumeTimes > maxReconsumeTimes) {
+                properties.put(MessageConst.PROPERTY_DELAY_TIME_LEVEL, "-1");
                 newTopic = MixAll.getDLQTopic(groupName);
-                int queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % DLQ_NUMS_PER_GROUP;
+                int queueIdInt = randomQueueId(DLQ_NUMS_PER_GROUP);
                 topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(newTopic,
                     DLQ_NUMS_PER_GROUP,
                     PermName.PERM_WRITE | PermName.PERM_READ, 0
@@ -386,64 +203,61 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         return true;
     }
 
-    private RemotingCommand sendMessage(final ChannelHandlerContext ctx,
-                                        final RemotingCommand request,
-                                        final SendMessageContext sendMessageContext,
-                                        final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
+    public RemotingCommand sendMessage(final ChannelHandlerContext ctx,
+        final RemotingCommand request,
+        final SendMessageContext sendMessageContext,
+        final SendMessageRequestHeader requestHeader,
+        final TopicQueueMappingContext mappingContext,
+        final SendMessageCallback sendMessageCallback) throws RemotingCommandException {
 
-        final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
-        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
-
-        response.setOpaque(request.getOpaque());
-
-        response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
-        response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
-
-        log.debug("receive SendMessage request command, {}", request);
-
-        final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
-        if (this.brokerController.getMessageStore().now() < startTimstamp) {
-            response.setCode(ResponseCode.SYSTEM_ERROR);
-            response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp)));
-            return response;
-        }
-
-        response.setCode(-1);
-        super.msgCheck(ctx, requestHeader, response);
+        final RemotingCommand response = preSend(ctx, request, requestHeader);
         if (response.getCode() != -1) {
             return response;
         }
 
+        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();
+
         final byte[] body = request.getBody();
 
         int queueIdInt = requestHeader.getQueueId();
         TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
 
         if (queueIdInt < 0) {
-            queueIdInt = ThreadLocalRandom.current().nextInt(99999999) % topicConfig.getWriteQueueNums();
+            queueIdInt = randomQueueId(topicConfig.getWriteQueueNums());
         }
 
         MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
         msgInner.setTopic(requestHeader.getTopic());
         msgInner.setQueueId(queueIdInt);
 
-        if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig)) {
+        Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+        if (!handleRetryAndDLQ(requestHeader, response, request, msgInner, topicConfig, oriProps)) {
             return response;
         }
 
         msgInner.setBody(body);
         msgInner.setFlag(requestHeader.getFlag());
-        MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties()));
+
+        String uniqKey = oriProps.get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX);
+        if (uniqKey == null || uniqKey.length() <= 0) {
+            uniqKey = MessageClientIDSetter.createUniqID();
+            oriProps.put(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, uniqKey);
+        }
+
+        MessageAccessor.setProperties(msgInner, oriProps);
+        msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(), msgInner.getTags()));
         msgInner.setBornTimestamp(requestHeader.getBornTimestamp());
         msgInner.setBornHost(ctx.channel().remoteAddress());
         msgInner.setStoreHost(this.getStoreHost());
         msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes());
         String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
         MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName);
+
         msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
-        PutMessageResult putMessageResult = null;
-        Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
+
+        // Map<String, String> oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties());
         String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
+        boolean sendTransactionPrepareMessage = false;
         if (Boolean.parseBoolean(traFlag)
             && !(msgInner.getReconsumeTimes() > 0 && msgInner.getDelayTimeLevel() > 0)) { //For client under version 4.6.1
             if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) {
@@ -453,19 +267,49 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
                         + "] sending transaction message is forbidden");
                 return response;
             }
-            putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner);
-        } else {
-            putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+            sendTransactionPrepareMessage = true;
         }
 
-        return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt);
+        long beginTimeMillis = this.brokerController.getMessageStore().now();
 
+        if (brokerController.getBrokerConfig().isAsyncSendEnable()) {
+            CompletableFuture<PutMessageResult> asyncPutMessageFuture;
+            if (sendTransactionPrepareMessage) {
+                asyncPutMessageFuture = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner);
+            } else {
+                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessage(msgInner);
+            }
+
+            final int finalQueueIdInt = queueIdInt;
+            final MessageExtBrokerInner finalMsgInner = msgInner;
+            asyncPutMessageFuture.thenAcceptAsync(putMessageResult -> {
+                RemotingCommand responseFuture =
+                    handlePutMessageResult(putMessageResult, response, request, finalMsgInner, responseHeader, sendMessageContext,
+                        ctx, finalQueueIdInt, beginTimeMillis, mappingContext);
+                if (responseFuture != null) {
+                    doResponse(ctx, request, responseFuture);
+                }
+                sendMessageCallback.onComplete(sendMessageContext, response);
+            }, this.brokerController.getPutMessageFutureExecutor());
+            // Returns null to release the send message thread
+            return null;
+        } else {
+            PutMessageResult putMessageResult = null;
+            if (sendTransactionPrepareMessage) {
+                putMessageResult = this.brokerController.getTransactionalMessageService().prepareMessage(msgInner);
+            } else {
+                putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
+            }
+            handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, beginTimeMillis, mappingContext);
+            sendMessageCallback.onComplete(sendMessageContext, response);
+            return response;
+        }
     }
 
     private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response,
-                                                   RemotingCommand request, MessageExt msg,
-                                                   SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx,
-                                                   int queueIdInt) {
+        RemotingCommand request, MessageExt msg,
+        SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx,
+        int queueIdInt, long beginTimeMillis, TopicQueueMappingContext mappingContext) {
         if (putMessageResult == null) {
             response.setCode(ResponseCode.SYSTEM_ERROR);
             response.setRemark("store putMessage return null");
@@ -493,7 +337,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
                 break;
 
             // Failed
-            case CREATE_MAPEDFILE_FAILED:
+            case IN_SYNC_REPLICAS_NOT_ENOUGH:
+                response.setCode(ResponseCode.SYSTEM_ERROR);
+                response.setRemark("in-sync replicas not enough");
+                break;
+            case CREATE_MAPPED_FILE_FAILED:
                 response.setCode(ResponseCode.SYSTEM_ERROR);
                 response.setRemark("create mapped file failed, server is busy or broken.");
                 break;
@@ -509,7 +357,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
                     "service not available now. It may be caused by one of the following reasons: " +
                         "the broker's disk is full [" + diskUtil() + "], messages are put to the slave, message store has been shut down, etc.");
                 break;
-            case OS_PAGECACHE_BUSY:
+            case OS_PAGE_CACHE_BUSY:
                 response.setCode(ResponseCode.SYSTEM_ERROR);
                 response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while");
                 break;
@@ -528,6 +376,10 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         }
 
         String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
+        String authType = request.getExtFields().get(BrokerStatsManager.ACCOUNT_AUTH_TYPE);
+        String ownerParent = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_PARENT);
+        String ownerSelf = request.getExtFields().get(BrokerStatsManager.ACCOUNT_OWNER_SELF);
+        int commercialSizePerMsg = brokerController.getBrokerConfig().getCommercialSizePerMsg();
         if (sendOK) {
 
             if (TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msg.getTopic())) {
@@ -539,12 +391,22 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
             this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(),
                 putMessageResult.getAppendMessageResult().getWroteBytes());
             this.brokerController.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum());
+            this.brokerController.getBrokerStatsManager().incTopicPutLatency(msg.getTopic(), queueIdInt,
+                (int) (this.brokerController.getMessageStore().now() - beginTimeMillis));
 
             response.setRemark(null);
 
             responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId());
             responseHeader.setQueueId(queueIdInt);
             responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());
+            responseHeader.setTransactionId(MessageClientIDSetter.getUniqID(msg));
+
+            RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext);
+            if (rewriteResult != null) {
+                return rewriteResult;
+            }
+
+            doResponse(ctx, request, response);
 
             if (hasSendMessageHook()) {
                 sendMessageContext.setMsgId(responseHeader.getMsgId());
@@ -553,36 +415,62 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
 
                 int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount();
                 int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes();
-                int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount;
+                int msgNum = putMessageResult.getAppendMessageResult().getMsgNum();
+                int commercialMsgNum = (int) Math.ceil(wroteSize / (double) commercialSizePerMsg);
+                int incValue = commercialMsgNum * commercialBaseCount;
 
                 sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS);
                 sendMessageContext.setCommercialSendTimes(incValue);
                 sendMessageContext.setCommercialSendSize(wroteSize);
                 sendMessageContext.setCommercialOwner(owner);
+
+                sendMessageContext.setSendStat(BrokerStatsManager.StatsType.SEND_SUCCESS);
+                sendMessageContext.setCommercialSendMsgNum(commercialMsgNum);
+                sendMessageContext.setAccountAuthType(authType);
+                sendMessageContext.setAccountOwnerParent(ownerParent);
+                sendMessageContext.setAccountOwnerSelf(ownerSelf);
+                sendMessageContext.setSendMsgSize(wroteSize);
+                sendMessageContext.setSendMsgNum(msgNum);
             }
-            return response;
+            return null;
         } else {
             if (hasSendMessageHook()) {
+                AppendMessageResult appendMessageResult = putMessageResult.getAppendMessageResult();
+
+                // TODO process partial failures of batch message
                 int wroteSize = request.getBody().length;
-                int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT);
+                int msgNum = Math.max(appendMessageResult != null ? appendMessageResult.getMsgNum() : 1, 1);
+                int commercialMsgNum = (int) Math.ceil(wroteSize / (double) commercialSizePerMsg);
+                int incValue = commercialMsgNum;
 
                 sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE);
                 sendMessageContext.setCommercialSendTimes(incValue);
                 sendMessageContext.setCommercialSendSize(wroteSize);
                 sendMessageContext.setCommercialOwner(owner);
+
+                sendMessageContext.setSendStat(BrokerStatsManager.StatsType.SEND_FAILURE);
+                sendMessageContext.setCommercialSendMsgNum(commercialMsgNum);
+                sendMessageContext.setAccountAuthType(authType);
+                sendMessageContext.setAccountOwnerParent(ownerParent);
+                sendMessageContext.setAccountOwnerSelf(ownerSelf);
+                sendMessageContext.setSendMsgSize(wroteSize);
+                sendMessageContext.setSendMsgNum(msgNum);
             }
         }
         return response;
     }
 
-    private CompletableFuture<RemotingCommand> asyncSendBatchMessage(ChannelHandlerContext ctx, RemotingCommand request,
-                                                                     SendMessageContext mqtraceContext,
-                                                                     SendMessageRequestHeader requestHeader) {
+    private RemotingCommand sendBatchMessage(final ChannelHandlerContext ctx,
+        final RemotingCommand request,
+        final SendMessageContext sendMessageContext,
+        final SendMessageRequestHeader requestHeader,
+        TopicQueueMappingContext mappingContext,
+        final SendMessageCallback sendMessageCallback) {
         final RemotingCommand response = preSend(ctx, request, requestHeader);
-        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
+        final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader();
 
         if (response.getCode() != -1) {
-            return CompletableFuture.completedFuture(response);
+            return response;
         }
 
         int queueIdInt = requestHeader.getQueueId();
@@ -595,9 +483,14 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         if (requestHeader.getTopic().length() > Byte.MAX_VALUE) {
             response.setCode(ResponseCode.MESSAGE_ILLEGAL);
             response.setRemark("message topic length too long " + requestHeader.getTopic().length());
-            return CompletableFuture.completedFuture(response);
+            return response;
         }
 
+        if (requestHeader.getTopic() != null && requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+            response.setCode(ResponseCode.MESSAGE_ILLEGAL);
+            response.setRemark("batch request does not support retry group " + requestHeader.getTopic());
+            return response;
+        }
         MessageExtBatch messageExtBatch = new MessageExtBatch();
         messageExtBatch.setTopic(requestHeader.getTopic());
         messageExtBatch.setQueueId(queueIdInt);
@@ -618,29 +511,57 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
         MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName);
 
-        CompletableFuture<PutMessageResult> putMessageResult = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch);
-        return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt);
-    }
+        boolean isInnerBatch = false;
 
-    public boolean hasConsumeMessageHook() {
-        return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty();
-    }
+        if (QueueTypeUtils.isBatchCq(Optional.of(topicConfig)) && MessageClientIDSetter.getUniqID(messageExtBatch) != null) {
+            // newly introduced inner-batch message
+            messageExtBatch.setSysFlag(messageExtBatch.getSysFlag() | MessageSysFlag.NEED_UNWRAP_FLAG);
+            messageExtBatch.setSysFlag(messageExtBatch.getSysFlag() | MessageSysFlag.INNER_BATCH_FLAG);
+            messageExtBatch.setInnerBatch(true);
 
-    public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) {
-        if (hasConsumeMessageHook()) {
-            for (ConsumeMessageHook hook : this.consumeMessageHookList) {
-                try {
-                    hook.consumeMessageAfter(context);
-                } catch (Throwable e) {
-                    // Ignore
-                }
-            }
+            int innerNum = MessageDecoder.countInnerMsgNum(ByteBuffer.wrap(messageExtBatch.getBody()));
+
+            MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_INNER_NUM, String.valueOf(innerNum));
+            messageExtBatch.setPropertiesString(MessageDecoder.messageProperties2String(messageExtBatch.getProperties()));
+
+            // tell the producer that it's an inner-batch message response.
+            responseHeader.setBatchUniqId(MessageClientIDSetter.getUniqID(messageExtBatch));
+
+            isInnerBatch = true;
         }
-    }
 
-    @Override
-    public SocketAddress getStoreHost() {
-        return storeHost;
+        long beginTimeMillis = this.brokerController.getMessageStore().now();
+
+        if (this.brokerController.getBrokerConfig().isAsyncSendEnable()) {
+            CompletableFuture<PutMessageResult> asyncPutMessageFuture;
+            if (isInnerBatch) {
+                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessage(messageExtBatch);
+            } else {
+                asyncPutMessageFuture = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch);
+            }
+            final int finalQueueIdInt = queueIdInt;
+            asyncPutMessageFuture.thenAcceptAsync(putMessageResult -> {
+                RemotingCommand responseFuture =
+                    handlePutMessageResult(putMessageResult, response, request, messageExtBatch, responseHeader,
+                        sendMessageContext, ctx, finalQueueIdInt, beginTimeMillis, mappingContext);
+                if (responseFuture != null) {
+                    doResponse(ctx, request, responseFuture);
+                }
+                sendMessageCallback.onComplete(sendMessageContext, response);
+            }, this.brokerController.getSendMessageExecutor());
+            // Returns null to release the send message thread
+            return null;
+        } else {
+            PutMessageResult putMessageResult;
+            if (isInnerBatch) {
+                putMessageResult = this.brokerController.getMessageStore().putMessage(messageExtBatch);
+            } else {
+                putMessageResult = this.brokerController.getMessageStore().putMessages(messageExtBatch);
+            }
+            handlePutMessageResult(putMessageResult, response, request, messageExtBatch, responseHeader, sendMessageContext, ctx, queueIdInt, beginTimeMillis, mappingContext);
+            sendMessageCallback.onComplete(sendMessageContext, response);
+            return response;
+        }
     }
 
     private String diskUtil() {
@@ -652,7 +573,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         } else {
             storePath = this.brokerController.getMessageStoreConfig().getStorePathCommitLog();
         }
-        String[] paths = storePath.trim().split(MessageStoreConfig.MULTI_PATH_SPLITTER);
+        String[] paths = storePath.trim().split(MixAll.MULTI_PATH_SPLITTER);
         for (String storePathPhysic : paths) {
             physicRatio = Math.min(physicRatio, UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic));
         }
@@ -668,29 +589,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         return String.format("CL: %5.2f CQ: %5.2f INDEX: %5.2f", physicRatio, logisRatio, indexRatio);
     }
 
-    public void registerConsumeMessageHook(List<ConsumeMessageHook> consumeMessageHookList) {
-        this.consumeMessageHookList = consumeMessageHookList;
-    }
-
-    static private ConsumeMessageContext buildConsumeMessageContext(String namespace,
-                                                                    ConsumerSendMsgBackRequestHeader requestHeader,
-                                                                    RemotingCommand request) {
-        ConsumeMessageContext context = new ConsumeMessageContext();
-        context.setNamespace(namespace);
-        context.setConsumerGroup(requestHeader.getGroup());
-        context.setTopic(requestHeader.getOriginTopic());
-        context.setCommercialRcvStats(BrokerStatsManager.StatsType.SEND_BACK);
-        context.setCommercialRcvTimes(1);
-        context.setCommercialOwner(request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER));
-        return context;
-    }
-
-    private int randomQueueId(int writeQueueNums) {
-        return ThreadLocalRandom.current().nextInt(99999999) % writeQueueNums;
-    }
-
     private RemotingCommand preSend(ChannelHandlerContext ctx, RemotingCommand request,
-                                    SendMessageRequestHeader requestHeader) {
+        SendMessageRequestHeader requestHeader) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
 
         response.setOpaque(request.getOpaque());
@@ -698,7 +598,7 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId());
         response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn()));
 
-        log.debug("Receive SendMessage request command {}", request);
+        LOGGER.debug("Receive SendMessage request command {}", request);
 
         final long startTimestamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp();
 
@@ -709,8 +609,9 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor {
         }
 
         response.setCode(-1);
-        super.msgCheck(ctx, requestHeader, response);
+        super.msgCheck(ctx, requestHeader, request, response);
 
         return response;
     }
+
 }

broker/src/main/java/org/apache/rocketmq/broker/schedule/DelayOffsetSerializeWrapper.java

@@ -14,16 +14,19 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.store.schedule;
+package org.apache.rocketmq.broker.schedule;
 
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import org.apache.rocketmq.common.DataVersion;
 import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 
 public class DelayOffsetSerializeWrapper extends RemotingSerializable {
     private ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =
         new ConcurrentHashMap<Integer, Long>(32);
 
+    private DataVersion dataVersion;
+
     public ConcurrentMap<Integer, Long> getOffsetTable() {
         return offsetTable;
     }
@@ -31,4 +34,12 @@ public class DelayOffsetSerializeWrapper extends RemotingSerializable {
     public void setOffsetTable(ConcurrentMap<Integer, Long> offsetTable) {
         this.offsetTable = offsetTable;
     }
+
+    public DataVersion getDataVersion() {
+        return dataVersion;
+    }
+
+    public void setDataVersion(DataVersion dataVersion) {
+        this.dataVersion = dataVersion;
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java

@@ -14,10 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocketmq.store.schedule;
+package org.apache.rocketmq.broker.schedule;
 
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.CompletableFuture;
@@ -29,12 +31,18 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.DataVersion;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.ThreadUtils;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.message.MessageAccessor;
@@ -42,15 +50,12 @@ import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.running.RunningStats;
-import org.apache.rocketmq.store.ConsumeQueue;
-import org.apache.rocketmq.store.ConsumeQueueExt;
-import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
-import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
-import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
+import org.apache.rocketmq.store.queue.CqUnit;
+import org.apache.rocketmq.store.queue.ReferredIterator;
 
 public class ScheduleMessageService extends ConfigManager {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
@@ -66,22 +71,33 @@ public class ScheduleMessageService extends ConfigManager {
 
     private final ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =
         new ConcurrentHashMap<Integer, Long>(32);
-    private final DefaultMessageStore defaultMessageStore;
     private final AtomicBoolean started = new AtomicBoolean(false);
     private ScheduledExecutorService deliverExecutorService;
-    private MessageStore writeMessageStore;
     private int maxDelayLevel;
+    private DataVersion dataVersion = new DataVersion();
     private boolean enableAsyncDeliver = false;
     private ScheduledExecutorService handleExecutorService;
+    private final ScheduledExecutorService scheduledPersistService;
     private final Map<Integer /* level */, LinkedBlockingQueue<PutResultProcess>> deliverPendingTable =
         new ConcurrentHashMap<>(32);
+    private final BrokerController brokerController;
+    private final transient AtomicLong versionChangeCounter = new AtomicLong(0);
 
-    public ScheduleMessageService(final DefaultMessageStore defaultMessageStore) {
-        this.defaultMessageStore = defaultMessageStore;
-        this.writeMessageStore = defaultMessageStore;
-        if (defaultMessageStore != null) {
-            this.enableAsyncDeliver = defaultMessageStore.getMessageStoreConfig().isEnableScheduleAsyncDeliver();
-        }
+    public ScheduleMessageService(final BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver();
+        scheduledPersistService = new ScheduledThreadPoolExecutor(1,
+            new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig()));
+        scheduledPersistService.scheduleAtFixedRate(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    ScheduleMessageService.this.persist();
+                } catch (Throwable e) {
+                    log.error("scheduleAtFixedRate flush exception", e);
+                }
+            }
+        }, 10000, this.brokerController.getMessageStoreConfig().getFlushDelayOffsetInterval(), TimeUnit.MILLISECONDS);
     }
 
     public static int queueId2DelayLevel(final int queueId) {
@@ -92,20 +108,13 @@ public class ScheduleMessageService extends ConfigManager {
         return delayLevel - 1;
     }
 
-    /**
-     * @param writeMessageStore the writeMessageStore to set
-     */
-    public void setWriteMessageStore(MessageStore writeMessageStore) {
-        this.writeMessageStore = writeMessageStore;
-    }
-
     public void buildRunningStats(HashMap<String, String> stats) {
         Iterator<Map.Entry<Integer, Long>> it = this.offsetTable.entrySet().iterator();
         while (it.hasNext()) {
             Map.Entry<Integer, Long> next = it.next();
             int queueId = delayLevel2QueueId(next.getKey());
             long delayOffset = next.getValue();
-            long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, queueId);
+            long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, queueId);
             String value = String.format("%d,%d", delayOffset, maxOffset);
             String key = String.format("%s_%d", RunningStats.scheduleMessageOffset.name(), next.getKey());
             stats.put(key, value);
@@ -114,6 +123,10 @@ public class ScheduleMessageService extends ConfigManager {
 
     private void updateOffset(int delayLevel, long offset) {
         this.offsetTable.put(delayLevel, offset);
+        if (versionChangeCounter.incrementAndGet() % brokerController.getBrokerConfig().getDelayOffsetUpdateVersionStep() == 0) {
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
+        }
     }
 
     public long computeDeliverTimestamp(final int delayLevel, final long storeTimestamp) {
@@ -160,11 +173,16 @@ public class ScheduleMessageService extends ConfigManager {
                         log.error("scheduleAtFixedRate flush exception", e);
                     }
                 }
-            }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval(), TimeUnit.MILLISECONDS);
+            }, 10000, this.brokerController.getMessageStore().getMessageStoreConfig().getFlushDelayOffsetInterval(), TimeUnit.MILLISECONDS);
         }
     }
 
     public void shutdown() {
+        stop();
+        ThreadUtils.shutdown(scheduledPersistService);
+    }
+
+    public void stop() {
         if (this.started.compareAndSet(true, false) && null != this.deliverExecutorService) {
             this.deliverExecutorService.shutdown();
             try {
@@ -200,6 +218,14 @@ public class ScheduleMessageService extends ConfigManager {
         return maxDelayLevel;
     }
 
+    public DataVersion getDataVersion() {
+        return dataVersion;
+    }
+
+    public void setDataVersion(DataVersion dataVersion) {
+        this.dataVersion = dataVersion;
+    }
+
     @Override
     public String encode() {
         return this.encode(false);
@@ -216,8 +242,8 @@ public class ScheduleMessageService extends ConfigManager {
     public boolean correctDelayOffset() {
         try {
             for (int delayLevel : delayLevelTable.keySet()) {
-                ConsumeQueue cq =
-                    ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
+                ConsumeQueueInterface cq =
+                    brokerController.getMessageStore().getQueueStore().findOrCreateConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
                         delayLevel2QueueId(delayLevel));
                 Long currentDelayOffset = offsetTable.get(delayLevel);
                 if (currentDelayOffset == null || cq == null) {
@@ -251,7 +277,7 @@ public class ScheduleMessageService extends ConfigManager {
 
     @Override
     public String configFilePath() {
-        return StorePathConfigHelper.getDelayOffsetStorePath(this.defaultMessageStore.getMessageStoreConfig()
+        return StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController.getMessageStore().getMessageStoreConfig()
             .getStorePathRootDir());
     }
 
@@ -262,6 +288,10 @@ public class ScheduleMessageService extends ConfigManager {
                 DelayOffsetSerializeWrapper.fromJson(jsonString, DelayOffsetSerializeWrapper.class);
             if (delayOffsetSerializeWrapper != null) {
                 this.offsetTable.putAll(delayOffsetSerializeWrapper.getOffsetTable());
+                // For compatible
+                if (delayOffsetSerializeWrapper.getDataVersion() != null) {
+                    this.dataVersion.assignNewOne(delayOffsetSerializeWrapper.getDataVersion());
+                }
             }
         }
     }
@@ -270,6 +300,7 @@ public class ScheduleMessageService extends ConfigManager {
     public String encode(final boolean prettyFormat) {
         DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = new DelayOffsetSerializeWrapper();
         delayOffsetSerializeWrapper.setOffsetTable(this.offsetTable);
+        delayOffsetSerializeWrapper.setDataVersion(this.dataVersion);
         return delayOffsetSerializeWrapper.toJson(prettyFormat);
     }
 
@@ -280,7 +311,7 @@ public class ScheduleMessageService extends ConfigManager {
         timeUnitTable.put("h", 1000L * 60 * 60);
         timeUnitTable.put("d", 1000L * 60 * 60 * 24);
 
-        String levelString = this.defaultMessageStore.getMessageStoreConfig().getMessageDelayLevel();
+        String levelString = this.brokerController.getMessageStoreConfig().getMessageDelayLevel();
         try {
             String[] levelArray = levelString.split(" ");
             for (int i = 0; i < levelArray.length; i++) {
@@ -338,6 +369,17 @@ public class ScheduleMessageService extends ConfigManager {
         return msgInner;
     }
 
+    public int computeDelayLevel(long timeMillis) {
+        long intervalMillis = timeMillis - System.currentTimeMillis();
+        List<Map.Entry<Integer, Long>> sortedLevels = delayLevelTable.entrySet().stream().sorted(Comparator.comparingLong(Map.Entry::getValue)).collect(Collectors.toList());
+        for (Map.Entry<Integer, Long> entry : sortedLevels) {
+            if (entry.getValue() > intervalMillis) {
+                return entry.getKey();
+            }
+        }
+        return sortedLevels.get(sortedLevels.size() - 1).getKey();
+    }
+
     class DeliverDelayedMessageTimerTask implements Runnable {
         private final int delayLevel;
         private final long offset;
@@ -376,8 +418,8 @@ public class ScheduleMessageService extends ConfigManager {
         }
 
         public void executeOnTimeup() {
-            ConsumeQueue cq =
-                ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
+            ConsumeQueueInterface cq =
+                ScheduleMessageService.this.brokerController.getMessageStore().getConsumeQueue(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
                     delayLevel2QueueId(delayLevel));
 
             if (cq == null) {
@@ -385,7 +427,7 @@ public class ScheduleMessageService extends ConfigManager {
                 return;
             }
 
-            SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(this.offset);
+            ReferredIterator<CqUnit> bufferCQ = cq.iterateFrom(this.offset);
             if (bufferCQ == null) {
                 long resetOffset;
                 if ((resetOffset = cq.getMinOffsetInQueue()) > this.offset) {
@@ -404,36 +446,35 @@ public class ScheduleMessageService extends ConfigManager {
 
             long nextOffset = this.offset;
             try {
-                int i = 0;
-                ConsumeQueueExt.CqExtUnit cqExtUnit = new ConsumeQueueExt.CqExtUnit();
-                for (; i < bufferCQ.getSize() && isStarted(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) {
-                    long offsetPy = bufferCQ.getByteBuffer().getLong();
-                    int sizePy = bufferCQ.getByteBuffer().getInt();
-                    long tagsCode = bufferCQ.getByteBuffer().getLong();
+                while (bufferCQ.hasNext() && isStarted()) {
+                    CqUnit cqUnit = bufferCQ.next();
+                    long offsetPy = cqUnit.getPos();
+                    int sizePy = cqUnit.getSize();
+                    long tagsCode = cqUnit.getTagsCode();
 
-                    if (cq.isExtAddr(tagsCode)) {
-                        if (cq.getExt(tagsCode, cqExtUnit)) {
-                            tagsCode = cqExtUnit.getTagsCode();
-                        } else {
-                            //can't find ext content.So re compute tags code.
-                            log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}",
-                                tagsCode, offsetPy, sizePy);
-                            long msgStoreTime = defaultMessageStore.getCommitLog().pickupStoreTimestamp(offsetPy, sizePy);
-                            tagsCode = computeDeliverTimestamp(delayLevel, msgStoreTime);
-                        }
+                    if (!cqUnit.isTagsCodeValid()) {
+                        //can't find ext content.So re compute tags code.
+                        log.error("[BUG] can't find consume queue extend file content!addr={}, offsetPy={}, sizePy={}",
+                            tagsCode, offsetPy, sizePy);
+                        long msgStoreTime = ScheduleMessageService.this.brokerController.getMessageStore().getCommitLog().pickupStoreTimestamp(offsetPy, sizePy);
+                        tagsCode = computeDeliverTimestamp(delayLevel, msgStoreTime);
                     }
 
                     long now = System.currentTimeMillis();
                     long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode);
-                    nextOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
+
+                    long currOffset = cqUnit.getQueueOffset();
+                    assert cqUnit.getBatchNum() == 1;
+                    nextOffset = currOffset + cqUnit.getBatchNum();
 
                     long countdown = deliverTimestamp - now;
                     if (countdown > 0) {
-                        this.scheduleNextTimerTask(nextOffset, DELAY_FOR_A_WHILE);
+                        this.scheduleNextTimerTask(currOffset, DELAY_FOR_A_WHILE);
+                        ScheduleMessageService.this.updateOffset(this.delayLevel, currOffset);
                         return;
                     }
 
-                    MessageExt msgExt = ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset(offsetPy, sizePy);
+                    MessageExt msgExt = ScheduleMessageService.this.brokerController.getMessageStore().lookMessageByOffset(offsetPy, sizePy);
                     if (msgExt == null) {
                         continue;
                     }
@@ -447,9 +488,9 @@ public class ScheduleMessageService extends ConfigManager {
 
                     boolean deliverSuc;
                     if (ScheduleMessageService.this.enableAsyncDeliver) {
-                        deliverSuc = this.asyncDeliver(msgInner, msgExt.getMsgId(), nextOffset, offsetPy, sizePy);
+                        deliverSuc = this.asyncDeliver(msgInner, msgExt.getMsgId(), currOffset, offsetPy, sizePy);
                     } else {
-                        deliverSuc = this.syncDeliver(msgInner, msgExt.getMsgId(), nextOffset, offsetPy, sizePy);
+                        deliverSuc = this.syncDeliver(msgInner, msgExt.getMsgId(), currOffset, offsetPy, sizePy);
                     }
 
                     if (!deliverSuc) {
@@ -457,8 +498,6 @@ public class ScheduleMessageService extends ConfigManager {
                         return;
                     }
                 }
-
-                nextOffset = this.offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE);
             } catch (Exception e) {
                 log.error("ScheduleMessageService, messageTimeup execute error, offset = {}", nextOffset, e);
             } finally {
@@ -490,7 +529,7 @@ public class ScheduleMessageService extends ConfigManager {
 
             //Flow Control
             int currentPendingNum = processesQueue.size();
-            int maxPendingLimit = ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig()
+            int maxPendingLimit = brokerController.getMessageStoreConfig()
                 .getScheduleAsyncDeliverMaxPendingLimit();
             if (currentPendingNum > maxPendingLimit) {
                 log.warn("Asynchronous deliver triggers flow control, " +
@@ -513,7 +552,7 @@ public class ScheduleMessageService extends ConfigManager {
         private PutResultProcess deliverMessage(MessageExtBrokerInner msgInner, String msgId, long offset,
             long offsetPy, int sizePy, boolean autoResend) {
             CompletableFuture<PutMessageResult> future =
-                ScheduleMessageService.this.writeMessageStore.asyncPutMessage(msgInner);
+                brokerController.getEscapeBridge().asyncPutMessage(msgInner);
             return new PutResultProcess()
                 .setTopic(msgInner.getTopic())
                 .setDelayLevel(this.delayLevel)
@@ -694,14 +733,14 @@ public class ScheduleMessageService extends ConfigManager {
 
         public void onSuccess(PutMessageResult result) {
             this.status = ProcessStatus.SUCCESS;
-            if (ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig().isEnableScheduleMessageStats()) {
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incQueueGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getMsgNum());
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incQueueGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getWroteBytes());
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incGroupGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getMsgNum());
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incGroupGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getWroteBytes());
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutNums(this.topic, result.getAppendMessageResult().getMsgNum(), 1);
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutSize(this.topic, result.getAppendMessageResult().getWroteBytes());
-                ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incBrokerPutNums(result.getAppendMessageResult().getMsgNum());
+            if (ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig().isEnableScheduleMessageStats() && !result.isRemotePut()) {
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incQueueGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getMsgNum());
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incQueueGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, delayLevel - 1, result.getAppendMessageResult().getWroteBytes());
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incGroupGetNums(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getMsgNum());
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incGroupGetSize(MixAll.SCHEDULE_CONSUMER_GROUP, TopicValidator.RMQ_SYS_SCHEDULE_TOPIC, result.getAppendMessageResult().getWroteBytes());
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incTopicPutNums(this.topic, result.getAppendMessageResult().getMsgNum(), 1);
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incTopicPutSize(this.topic, result.getAppendMessageResult().getWroteBytes());
+                ScheduleMessageService.this.brokerController.getBrokerStatsManager().incBrokerPutNums(result.getAppendMessageResult().getMsgNum());
             }
         }
 
@@ -737,7 +776,7 @@ public class ScheduleMessageService extends ConfigManager {
             }
 
             try {
-                MessageExt msgExt = ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset(this.physicOffset, this.physicSize);
+                MessageExt msgExt = ScheduleMessageService.this.brokerController.getMessageStore().lookMessageByOffset(this.physicOffset, this.physicSize);
                 if (msgExt == null) {
                     log.warn("ScheduleMessageService resend not found message. info: {}", this.toString());
                     this.status = need2Skip() ? ProcessStatus.SKIP : ProcessStatus.EXCEPTION;
@@ -745,7 +784,7 @@ public class ScheduleMessageService extends ConfigManager {
                 }
 
                 MessageExtBrokerInner msgInner = ScheduleMessageService.this.messageTimeup(msgExt);
-                PutMessageResult result = ScheduleMessageService.this.writeMessageStore.putMessage(msgInner);
+                PutMessageResult result = ScheduleMessageService.this.brokerController.getEscapeBridge().putMessage(msgInner);
                 this.handleResult(result);
                 if (result != null && result.getPutMessageStatus() == PutMessageStatus.PUT_OK) {
                     log.info("Resend message success, info: {}", this.toString());
@@ -757,13 +796,13 @@ public class ScheduleMessageService extends ConfigManager {
         }
 
         public boolean need2Blocked() {
-            int maxResendNum2Blocked = ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig()
+            int maxResendNum2Blocked = ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig()
                 .getScheduleAsyncDeliverMaxResendNum2Blocked();
             return this.resendCount > maxResendNum2Blocked;
         }
 
         public boolean need2Skip() {
-            int maxResendNum2Blocked = ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig()
+            int maxResendNum2Blocked = ScheduleMessageService.this.brokerController.getMessageStore().getMessageStoreConfig()
                 .getScheduleAsyncDeliverMaxResendNum2Blocked();
             return this.resendCount > maxResendNum2Blocked * 2;
         }
@@ -805,4 +844,8 @@ public class ScheduleMessageService extends ConfigManager {
          */
         SKIP,
     }
+
+    public ConcurrentMap<Integer, Long> getOffsetTable() {
+        return offsetTable;
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java

@@ -16,20 +16,24 @@
  */
 package org.apache.rocketmq.broker.slave;
 
-import java.io.IOException;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.loadbalance.MessageRequestModeManager;
 import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.common.protocol.body.MessageRequestModeSerializeWrapper;
 import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
 import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
-import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
+import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
 
+import java.io.IOException;
+
 public class SlaveSynchronize {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private final BrokerController brokerController;
     private volatile String masterAddr = null;
 
@@ -42,7 +46,10 @@ public class SlaveSynchronize {
     }
 
     public void setMasterAddr(String masterAddr) {
-        this.masterAddr = masterAddr;
+        if (!StringUtils.equals(this.masterAddr, masterAddr)) {
+            this.masterAddr = masterAddr;
+            LOGGER.info("Update master address from {} to {}", this.masterAddr, masterAddr);
+        }
     }
 
     public void syncAll() {
@@ -50,13 +57,14 @@ public class SlaveSynchronize {
         this.syncConsumerOffset();
         this.syncDelayOffset();
         this.syncSubscriptionGroupConfig();
+        this.syncMessageRequestMode();
     }
 
     private void syncTopicConfig() {
         String masterAddrBak = this.masterAddr;
         if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
-                TopicConfigSerializeWrapper topicWrapper =
+                TopicConfigAndMappingSerializeWrapper topicWrapper =
                     this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);
                 if (!this.brokerController.getTopicConfigManager().getDataVersion()
                     .equals(topicWrapper.getDataVersion())) {
@@ -67,11 +75,19 @@ public class SlaveSynchronize {
                     this.brokerController.getTopicConfigManager().getTopicConfigTable()
                         .putAll(topicWrapper.getTopicConfigTable());
                     this.brokerController.getTopicConfigManager().persist();
-
-                    log.info("Update slave topic config from master, {}", masterAddrBak);
                 }
+                if (topicWrapper.getTopicQueueMappingDetailMap() != null
+                        && !topicWrapper.getMappingDataVersion().equals(this.brokerController.getTopicQueueMappingManager().getDataVersion())) {
+                    this.brokerController.getTopicQueueMappingManager().getDataVersion()
+                            .assignNewOne(topicWrapper.getMappingDataVersion());
+                    this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable().clear();
+                    this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable()
+                            .putAll(topicWrapper.getTopicQueueMappingDetailMap());
+                    this.brokerController.getTopicQueueMappingManager().persist();
+                }
+                LOGGER.info("Update slave topic config from master, {}", masterAddrBak);
             } catch (Exception e) {
-                log.error("SyncTopicConfig Exception, {}", masterAddrBak, e);
+                LOGGER.error("SyncTopicConfig Exception, {}", masterAddrBak, e);
             }
         }
     }
@@ -84,10 +100,11 @@ public class SlaveSynchronize {
                     this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak);
                 this.brokerController.getConsumerOffsetManager().getOffsetTable()
                     .putAll(offsetWrapper.getOffsetTable());
+                this.brokerController.getConsumerOffsetManager().getDataVersion().assignNewOne(offsetWrapper.getDataVersion());
                 this.brokerController.getConsumerOffsetManager().persist();
-                log.info("Update slave consumer offset from master, {}", masterAddrBak);
+                LOGGER.info("Update slave consumer offset from master, {}", masterAddrBak);
             } catch (Exception e) {
-                log.error("SyncConsumerOffset Exception, {}", masterAddrBak, e);
+                LOGGER.error("SyncConsumerOffset Exception, {}", masterAddrBak, e);
             }
         }
     }
@@ -105,13 +122,14 @@ public class SlaveSynchronize {
                             .getMessageStoreConfig().getStorePathRootDir());
                     try {
                         MixAll.string2File(delayOffset, fileName);
+                        this.brokerController.getScheduleMessageService().load();
                     } catch (IOException e) {
-                        log.error("Persist file Exception, {}", fileName, e);
+                        LOGGER.error("Persist file Exception, {}", fileName, e);
                     }
                 }
-                log.info("Update slave delay offset from master, {}", masterAddrBak);
+                LOGGER.info("Update slave delay offset from master, {}", masterAddrBak);
             } catch (Exception e) {
-                log.error("SyncDelayOffset Exception, {}", masterAddrBak, e);
+                LOGGER.error("SyncDelayOffset Exception, {}", masterAddrBak, e);
             }
         }
     }
@@ -134,10 +152,31 @@ public class SlaveSynchronize {
                     subscriptionGroupManager.getSubscriptionGroupTable().putAll(
                         subscriptionWrapper.getSubscriptionGroupTable());
                     subscriptionGroupManager.persist();
-                    log.info("Update slave Subscription Group from master, {}", masterAddrBak);
+                    LOGGER.info("Update slave Subscription Group from master, {}", masterAddrBak);
                 }
             } catch (Exception e) {
-                log.error("SyncSubscriptionGroup Exception, {}", masterAddrBak, e);
+                LOGGER.error("SyncSubscriptionGroup Exception, {}", masterAddrBak, e);
+            }
+        }
+    }
+
+    private void syncMessageRequestMode() {
+        String masterAddrBak = this.masterAddr;
+        if (masterAddrBak != null  && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
+            try {
+                MessageRequestModeSerializeWrapper messageRequestModeSerializeWrapper =
+                        this.brokerController.getBrokerOuterAPI().getAllMessageRequestMode(masterAddrBak);
+
+                MessageRequestModeManager messageRequestModeManager =
+                        this.brokerController.getQueryAssignmentProcessor().getMessageRequestModeManager();
+                messageRequestModeManager.getMessageRequestModeMap().clear();
+                messageRequestModeManager.getMessageRequestModeMap().putAll(
+                        messageRequestModeSerializeWrapper.getMessageRequestModeMap()
+                );
+                messageRequestModeManager.persist();
+                LOGGER.info("Update slave Message Request Mode from master, {}", masterAddrBak);
+            } catch (Exception e) {
+                LOGGER.error("SyncMessageRequestMode Exception, {}", masterAddrBak, e);
             }
         }
     }

broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java

@@ -36,6 +36,10 @@ public class SubscriptionGroupManager extends ConfigManager {
 
     private final ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroupTable =
         new ConcurrentHashMap<String, SubscriptionGroupConfig>(1024);
+
+    private final ConcurrentMap<String, ConcurrentMap<String, Integer>> forbiddenTable =
+        new ConcurrentHashMap<String, ConcurrentMap<String, Integer>>(4);
+
     private final DataVersion dataVersion = new DataVersion();
     private transient BrokerController brokerController;
 
@@ -94,6 +98,13 @@ public class SubscriptionGroupManager extends ConfigManager {
             subscriptionGroupConfig.setConsumeBroadcastEnable(true);
             this.subscriptionGroupTable.put(MixAll.CID_ONSAPI_OWNER_GROUP, subscriptionGroupConfig);
         }
+
+        {
+            SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig();
+            subscriptionGroupConfig.setGroupName(MixAll.CID_SYS_RMQ_TRANS);
+            subscriptionGroupConfig.setConsumeBroadcastEnable(true);
+            this.subscriptionGroupTable.put(MixAll.CID_SYS_RMQ_TRANS, subscriptionGroupConfig);
+        }
     }
 
     public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) {
@@ -104,6 +115,83 @@ public class SubscriptionGroupManager extends ConfigManager {
             log.info("create new subscription group, {}", config);
         }
 
+        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+        dataVersion.nextVersion(stateMachineVersion);
+
+        this.persist();
+    }
+
+    public void updateForbidden(String group, String topic, int forbiddenIndex, boolean setOrClear) {
+        if (setOrClear) {
+            setForbidden(group, topic, forbiddenIndex);
+        } else {
+            clearForbidden(group, topic, forbiddenIndex);
+        }
+    }
+
+    /**
+     * set the bit value to 1 at the specific index (from 0)
+     *
+     * @param group
+     * @param topic
+     * @param forbiddenIndex from 0
+     */
+    public void setForbidden(String group, String topic, int forbiddenIndex) {
+        int topicForbidden = getForbidden(group, topic);
+        topicForbidden |= 1 << forbiddenIndex;
+        updateForbiddenValue(group, topic, topicForbidden);
+    }
+
+    /**
+     * clear the bit value to 0 at the specific index (from 0)
+     *
+     * @param group
+     * @param topic
+     * @param forbiddenIndex from 0
+     */
+    public void clearForbidden(String group, String topic, int forbiddenIndex) {
+        int topicForbidden = getForbidden(group, topic);
+        topicForbidden &= ~(1 << forbiddenIndex);
+        updateForbiddenValue(group, topic, topicForbidden);
+    }
+
+    public boolean getForbidden(String group, String topic, int forbiddenIndex) {
+        int topicForbidden = getForbidden(group, topic);
+        int bitForbidden = 1 << forbiddenIndex;
+        return (topicForbidden & bitForbidden) == bitForbidden;
+    }
+
+    public int getForbidden(String group, String topic) {
+        ConcurrentMap<String, Integer> topicForbiddens = this.forbiddenTable.get(group);
+        if (topicForbiddens == null) {
+            return 0;
+        }
+        Integer topicForbidden = topicForbiddens.get(topic);
+        if (topicForbidden == null || topicForbidden < 0) {
+            topicForbidden = 0;
+        }
+        return topicForbidden;
+    }
+
+    private void updateForbiddenValue(String group, String topic, Integer forbidden) {
+        if (forbidden == null || forbidden <= 0) {
+            this.forbiddenTable.remove(group);
+            log.info("clear group forbidden, {}@{} ", group, topic);
+            return;
+        }
+
+        ConcurrentMap<String, Integer> topicsPermMap = this.forbiddenTable.get(group);
+        if (topicsPermMap == null) {
+            this.forbiddenTable.putIfAbsent(group, new ConcurrentHashMap<String, Integer>());
+            topicsPermMap = this.forbiddenTable.get(group);
+        }
+        Integer old = topicsPermMap.put(topic, forbidden);
+        if (old != null) {
+            log.info("set group forbidden, {}@{} old: {} new: {}", group, topic, old, forbidden);
+        } else {
+            log.info("set group forbidden, {}@{} old: {} new: {}", group, topic, 0, forbidden);
+        }
+
         this.dataVersion.nextVersion();
 
         this.persist();
@@ -113,7 +201,8 @@ public class SubscriptionGroupManager extends ConfigManager {
         SubscriptionGroupConfig old = this.subscriptionGroupTable.get(groupName);
         if (old != null) {
             old.setConsumeEnable(false);
-            this.dataVersion.nextVersion();
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
         }
     }
 
@@ -127,7 +216,8 @@ public class SubscriptionGroupManager extends ConfigManager {
                 if (null == preConfig) {
                     log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString());
                 }
-                this.dataVersion.nextVersion();
+                long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+                dataVersion.nextVersion(stateMachineVersion);
                 this.persist();
             }
         }
@@ -152,6 +242,9 @@ public class SubscriptionGroupManager extends ConfigManager {
             SubscriptionGroupManager obj = RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class);
             if (obj != null) {
                 this.subscriptionGroupTable.putAll(obj.subscriptionGroupTable);
+                if (obj.forbiddenTable != null) {
+                    this.forbiddenTable.putAll(obj.forbiddenTable);
+                }
                 this.dataVersion.assignNewOne(obj.dataVersion);
                 this.printLoadDataWhenFirstBoot(obj);
             }
@@ -174,18 +267,31 @@ public class SubscriptionGroupManager extends ConfigManager {
         return subscriptionGroupTable;
     }
 
+    public ConcurrentMap<String, ConcurrentMap<String, Integer>> getForbiddenTable() {
+        return forbiddenTable;
+    }
+
     public DataVersion getDataVersion() {
         return dataVersion;
     }
 
     public void deleteSubscriptionGroupConfig(final String groupName) {
         SubscriptionGroupConfig old = this.subscriptionGroupTable.remove(groupName);
+        this.forbiddenTable.remove(groupName);
         if (old != null) {
             log.info("delete subscription group OK, subscription group:{}", old);
-            this.dataVersion.nextVersion();
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
             this.persist();
         } else {
             log.warn("delete subscription group failed, subscription groupName: {} not exist", groupName);
         }
     }
+
+    public void setSubscriptionGroupTable(ConcurrentMap<String, SubscriptionGroupConfig> otherSubscriptionGroupTable) {
+        this.subscriptionGroupTable.clear();
+        for (String key : otherSubscriptionGroupTable.keySet()) {
+            this.subscriptionGroupTable.put(key, otherSubscriptionGroupTable.get(key));
+        }
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java

@@ -16,6 +16,8 @@
  */
 package org.apache.rocketmq.broker.topic;
 
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -25,11 +27,18 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.BrokerPathConfigHelper;
+import org.apache.rocketmq.common.PopAckConstants;
+import org.apache.rocketmq.common.attribute.Attribute;
 import org.apache.rocketmq.common.ConfigManager;
 import org.apache.rocketmq.common.DataVersion;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicAttributes;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
@@ -40,6 +49,8 @@ import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
 public class TopicConfigManager extends ConfigManager {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
     private static final long LOCK_TIMEOUT_MILLIS = 3000;
@@ -88,7 +99,6 @@ public class TopicConfigManager extends ConfigManager {
             this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
         }
         {
-
             String topic = this.brokerController.getBrokerConfig().getBrokerClusterName();
             TopicConfig topicConfig = new TopicConfig(topic);
             TopicValidator.addSystemTopic(topic);
@@ -147,6 +157,44 @@ public class TopicConfigManager extends ConfigManager {
             topicConfig.setWriteQueueNums(1);
             this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
         }
+        {
+            // PopAckConstants.REVIVE_TOPIC
+            String topic = PopAckConstants.buildClusterReviveTopic(this.brokerController.getBrokerConfig().getBrokerClusterName());
+            TopicConfig topicConfig = new TopicConfig(topic);
+            TopicValidator.addSystemTopic(topic);
+            topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig().getReviveQueueNum());
+            topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig().getReviveQueueNum());
+            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+        }
+        {
+            // sync broker member group topic
+            String topic = TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX + this.brokerController.getBrokerConfig().getBrokerName();
+            TopicConfig topicConfig = new TopicConfig(topic);
+            TopicValidator.addSystemTopic(topic);
+            topicConfig.setReadQueueNums(1);
+            topicConfig.setWriteQueueNums(1);
+            topicConfig.setPerm(PermName.PERM_INHERIT);
+            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+        }
+        {
+            // TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC
+            String topic = TopicValidator.RMQ_SYS_TRANS_HALF_TOPIC;
+            TopicConfig topicConfig = new TopicConfig(topic);
+            TopicValidator.addSystemTopic(topic);
+            topicConfig.setReadQueueNums(1);
+            topicConfig.setWriteQueueNums(1);
+            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+        }
+
+        {
+            // TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC
+            String topic = TopicValidator.RMQ_SYS_TRANS_OP_HALF_TOPIC;
+            TopicConfig topicConfig = new TopicConfig(topic);
+            TopicValidator.addSystemTopic(topic);
+            topicConfig.setReadQueueNums(1);
+            topicConfig.setWriteQueueNums(1);
+            this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+        }
     }
 
     public TopicConfig selectTopicConfig(final String topic) {
@@ -162,8 +210,9 @@ public class TopicConfigManager extends ConfigManager {
             if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
                 try {
                     topicConfig = this.topicConfigTable.get(topic);
-                    if (topicConfig != null)
+                    if (topicConfig != null) {
                         return topicConfig;
+                    }
 
                     TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic);
                     if (defaultTopicConfig != null) {
@@ -204,7 +253,8 @@ public class TopicConfigManager extends ConfigManager {
 
                         this.topicConfigTable.put(topic, topicConfig);
 
-                        this.dataVersion.nextVersion();
+                        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+                        dataVersion.nextVersion(stateMachineVersion);
 
                         createNew = true;
 
@@ -225,14 +275,66 @@ public class TopicConfigManager extends ConfigManager {
         return topicConfig;
     }
 
+    public TopicConfig createTopicIfAbsent(TopicConfig topicConfig) {
+        return createTopicIfAbsent(topicConfig, true);
+    }
+
+    public TopicConfig createTopicIfAbsent(TopicConfig topicConfig, boolean register) {
+        boolean createNew = false;
+        if (topicConfig == null) {
+            throw new NullPointerException("TopicConfig");
+        }
+        if (StringUtils.isEmpty(topicConfig.getTopicName())) {
+            throw new IllegalArgumentException("TopicName");
+        }
+
+        try {
+            if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
+                try {
+                    TopicConfig existedTopicConfig = this.topicConfigTable.get(topicConfig.getTopicName());
+                    if (existedTopicConfig != null) {
+                        return existedTopicConfig;
+                    }
+                    log.info("Create new topic [{}] config:[{}]", topicConfig.getTopicName(), topicConfig);
+                    this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
+                    this.dataVersion.nextVersion();
+                    createNew = true;
+                    this.persist();
+                } finally {
+                    this.topicConfigTableLock.unlock();
+                }
+            }
+        } catch (InterruptedException e) {
+            log.error("createTopicIfAbsent ", e);
+        }
+        if (createNew && register) {
+            this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion);
+        }
+        return this.topicConfigTable.get(topicConfig.getTopicName());
+    }
+
     public TopicConfig createTopicInSendMessageBackMethod(
         final String topic,
         final int clientDefaultTopicQueueNums,
         final int perm,
         final int topicSysFlag) {
+        return createTopicInSendMessageBackMethod(topic, clientDefaultTopicQueueNums, perm, false, topicSysFlag);
+    }
+
+    public TopicConfig createTopicInSendMessageBackMethod(
+        final String topic,
+        final int clientDefaultTopicQueueNums,
+        final int perm,
+        final boolean isOrder,
+        final int topicSysFlag) {
         TopicConfig topicConfig = this.topicConfigTable.get(topic);
-        if (topicConfig != null)
+        if (topicConfig != null) {
+            if (isOrder != topicConfig.isOrder()) {
+                topicConfig.setOrder(isOrder);
+                this.updateTopicConfig(topicConfig);
+            }
             return topicConfig;
+        }
 
         boolean createNew = false;
 
@@ -240,19 +342,22 @@ public class TopicConfigManager extends ConfigManager {
             if (this.topicConfigTableLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
                 try {
                     topicConfig = this.topicConfigTable.get(topic);
-                    if (topicConfig != null)
+                    if (topicConfig != null) {
                         return topicConfig;
+                    }
 
                     topicConfig = new TopicConfig(topic);
                     topicConfig.setReadQueueNums(clientDefaultTopicQueueNums);
                     topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums);
                     topicConfig.setPerm(perm);
                     topicConfig.setTopicSysFlag(topicSysFlag);
+                    topicConfig.setOrder(isOrder);
 
                     log.info("create new topic {}", topicConfig);
                     this.topicConfigTable.put(topic, topicConfig);
                     createNew = true;
-                    this.dataVersion.nextVersion();
+                    long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+                    dataVersion.nextVersion(stateMachineVersion);
                     this.persist();
                 } finally {
                     this.topicConfigTableLock.unlock();
@@ -325,7 +430,8 @@ public class TopicConfigManager extends ConfigManager {
 
             this.topicConfigTable.put(topic, topicConfig);
 
-            this.dataVersion.nextVersion();
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
 
             this.persist();
             this.brokerController.registerBrokerAll(false, true, true);
@@ -347,7 +453,8 @@ public class TopicConfigManager extends ConfigManager {
 
             this.topicConfigTable.put(topic, topicConfig);
 
-            this.dataVersion.nextVersion();
+            long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+            dataVersion.nextVersion(stateMachineVersion);
 
             this.persist();
             this.brokerController.registerBrokerAll(false, true, true);
@@ -355,6 +462,18 @@ public class TopicConfigManager extends ConfigManager {
     }
 
     public void updateTopicConfig(final TopicConfig topicConfig) {
+        checkNotNull(topicConfig, "topicConfig shouldn't be null");
+
+        Map<String, String> newAttributes = request(topicConfig);
+        Map<String, String> currentAttributes = current(topicConfig.getTopicName());
+
+        Map<String, String> finalAttributes = alterCurrentAttributes(
+            this.topicConfigTable.get(topicConfig.getTopicName()) == null,
+            ImmutableMap.copyOf(currentAttributes),
+            ImmutableMap.copyOf(newAttributes));
+
+        topicConfig.setAttributes(finalAttributes);
+
         TopicConfig old = this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig);
         if (old != null) {
             log.info("update topic config, old:[{}] new:[{}]", old, topicConfig);
@@ -362,9 +481,10 @@ public class TopicConfigManager extends ConfigManager {
             log.info("create new topic [{}]", topicConfig);
         }
 
-        this.dataVersion.nextVersion();
+        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+        dataVersion.nextVersion(stateMachineVersion);
 
-        this.persist();
+        this.persist(topicConfig.getTopicName(), topicConfig);
     }
 
     public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) {
@@ -381,7 +501,11 @@ public class TopicConfigManager extends ConfigManager {
                 }
             }
 
-            for (Map.Entry<String, TopicConfig> entry : this.topicConfigTable.entrySet()) {
+            // We don't have a mandatory rule to maintain the validity of order conf in NameServer,
+            // so we may overwrite the order field mistakenly.
+            // To avoid the above case, we comment the below codes, please use mqadmin API to update
+            // the order filed.
+            /*for (Map.Entry<String, TopicConfig> entry : this.topicConfigTable.entrySet()) {
                 String topic = entry.getKey();
                 if (!orderTopics.contains(topic)) {
                     TopicConfig topicConfig = entry.getValue();
@@ -391,15 +515,21 @@ public class TopicConfigManager extends ConfigManager {
                         log.info("update order topic config, topic={}, order={}", topic, false);
                     }
                 }
-            }
+            }*/
 
             if (isChange) {
-                this.dataVersion.nextVersion();
+                long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+                dataVersion.nextVersion(stateMachineVersion);
                 this.persist();
             }
         }
     }
 
+    // make it testable
+    public Map<String, Attribute> allAttributes() {
+        return TopicAttributes.ALL;
+    }
+
     public boolean isOrderTopic(final String topic) {
         TopicConfig topicConfig = this.topicConfigTable.get(topic);
         if (topicConfig == null) {
@@ -423,10 +553,18 @@ public class TopicConfigManager extends ConfigManager {
     public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() {
         TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
         topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable);
-        topicConfigSerializeWrapper.setDataVersion(this.dataVersion);
+        DataVersion dataVersionCopy = new DataVersion();
+        dataVersionCopy.assignNewOne(this.dataVersion);
+        topicConfigSerializeWrapper.setDataVersion(dataVersionCopy);
         return topicConfigSerializeWrapper;
     }
 
+    public void initStateVersion() {
+        long stateMachineVersion = brokerController.getMessageStore() != null ? brokerController.getMessageStore().getStateMachineVersion() : 0;
+        dataVersion.nextVersion(stateMachineVersion);
+        this.persist();
+    }
+
     @Override
     public String encode() {
         return encode(false);
@@ -434,8 +572,7 @@ public class TopicConfigManager extends ConfigManager {
 
     @Override
     public String configFilePath() {
-        return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig()
-            .getStorePathRootDir());
+        return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig().getStorePathRootDir());
     }
 
     @Override
@@ -473,4 +610,122 @@ public class TopicConfigManager extends ConfigManager {
     public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {
         return topicConfigTable;
     }
+
+    private Map<String, String> request(TopicConfig topicConfig) {
+        return topicConfig.getAttributes() == null ? new HashMap<>() : topicConfig.getAttributes();
+    }
+
+    private Map<String, String> current(String topic) {
+        TopicConfig topicConfig = this.topicConfigTable.get(topic);
+        if (topicConfig == null) {
+            return new HashMap<>();
+        } else {
+            Map<String, String> attributes = topicConfig.getAttributes();
+            if (attributes == null) {
+                return new HashMap<>();
+            } else {
+                return attributes;
+            }
+        }
+    }
+
+    private Map<String, String> alterCurrentAttributes(boolean create, ImmutableMap<String, String> currentAttributes,
+        ImmutableMap<String, String> newAttributes) {
+        Map<String, String> init = new HashMap<>();
+        Map<String, String> add = new HashMap<>();
+        Map<String, String> update = new HashMap<>();
+        Map<String, String> delete = new HashMap<>();
+        Set<String> keys = new HashSet<>();
+
+        for (Entry<String, String> attribute : newAttributes.entrySet()) {
+            String key = attribute.getKey();
+            String realKey = realKey(key);
+            String value = attribute.getValue();
+
+            validate(realKey);
+            duplicationCheck(keys, realKey);
+
+            if (create) {
+                if (key.startsWith("+")) {
+                    init.put(realKey, value);
+                } else {
+                    throw new RuntimeException("only add attribute is supported while creating topic. key: " + realKey);
+                }
+            } else {
+                if (key.startsWith("+")) {
+                    if (!currentAttributes.containsKey(realKey)) {
+                        add.put(realKey, value);
+                    } else {
+                        update.put(realKey, value);
+                    }
+                } else if (key.startsWith("-")) {
+                    if (!currentAttributes.containsKey(realKey)) {
+                        throw new RuntimeException("attempt to delete a nonexistent key: " + realKey);
+                    }
+                    delete.put(realKey, value);
+                } else {
+                    throw new RuntimeException("wrong format key: " + realKey);
+                }
+            }
+        }
+
+        validateAlter(init, true, false);
+        validateAlter(add, false, false);
+        validateAlter(update, false, false);
+        validateAlter(delete, false, true);
+
+        log.info("add: {}, update: {}, delete: {}", add, update, delete);
+        HashMap<String, String> finalAttributes = new HashMap<>(currentAttributes);
+        finalAttributes.putAll(init);
+        finalAttributes.putAll(add);
+        finalAttributes.putAll(update);
+        for (String s : delete.keySet()) {
+            finalAttributes.remove(s);
+        }
+        return finalAttributes;
+    }
+
+    private void duplicationCheck(Set<String> keys, String key) {
+        boolean notExist = keys.add(key);
+        if (!notExist) {
+            throw new RuntimeException("alter duplication key. key: " + key);
+        }
+    }
+
+    private void validate(String kvAttribute) {
+        if (Strings.isNullOrEmpty(kvAttribute)) {
+            throw new RuntimeException("kv string format wrong.");
+        }
+
+        if (kvAttribute.contains("+")) {
+            throw new RuntimeException("kv string format wrong.");
+        }
+
+        if (kvAttribute.contains("-")) {
+            throw new RuntimeException("kv string format wrong.");
+        }
+    }
+
+    private void validateAlter(Map<String, String> alter, boolean init, boolean delete) {
+        for (Entry<String, String> entry : alter.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue();
+
+            Attribute attribute = allAttributes().get(key);
+            if (attribute == null) {
+                throw new RuntimeException("unsupported key: " + key);
+            }
+            if (!init && !attribute.isChangeable()) {
+                throw new RuntimeException("attempt to update an unchangeable attribute. key: " + key);
+            }
+
+            if (!delete) {
+                attribute.verify(value);
+            }
+        }
+    }
+
+    private String realKey(String key) {
+        return key.substring(1);
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java

@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.topic;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.out.BrokerOuterAPI;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.ServiceThread;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.admin.TopicOffset;
+import org.apache.rocketmq.common.admin.TopicStatsTable;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.common.rpc.ClientMetadata;
+import org.apache.rocketmq.common.rpc.RpcClient;
+import org.apache.rocketmq.common.rpc.RpcRequest;
+import org.apache.rocketmq.common.rpc.RpcResponse;
+import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TopicQueueMappingCleanService extends ServiceThread {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    private TopicQueueMappingManager topicQueueMappingManager;
+    private BrokerOuterAPI brokerOuterAPI;
+    private RpcClient rpcClient;
+    private MessageStoreConfig messageStoreConfig;
+    private BrokerConfig brokerConfig;
+    private BrokerController brokerController;
+
+    public TopicQueueMappingCleanService(BrokerController brokerController) {
+        this.brokerController = brokerController;
+        this.topicQueueMappingManager = brokerController.getTopicQueueMappingManager();
+        this.rpcClient = brokerController.getBrokerOuterAPI().getRpcClient();
+        this.messageStoreConfig = brokerController.getMessageStoreConfig();
+        this.brokerConfig = brokerController.getBrokerConfig();
+        this.brokerOuterAPI = brokerController.getBrokerOuterAPI();
+    }
+
+    @Override
+    public String getServiceName() {
+        if (this.brokerConfig.isInBrokerContainer()) {
+            return this.brokerController.getBrokerIdentity().getLoggerIdentifier() + TopicQueueMappingCleanService.class.getSimpleName();
+        }
+        return TopicQueueMappingCleanService.class.getSimpleName();
+    }
+
+    @Override
+    public void run() {
+        log.info("Start topic queue mapping clean service thread!");
+        while (!this.isStopped()) {
+            try {
+                this.waitForRunning(5L * 60 * 1000);
+            } catch (Throwable ignored) {
+            }
+            try {
+                cleanItemExpired();
+            } catch (Throwable t) {
+                log.error("topic queue mapping cleanItemExpired failed", t);
+            }
+            try {
+                cleanItemListMoreThanSecondGen();
+            } catch (Throwable t) {
+                log.error("topic queue mapping cleanItemListMoreThanSecondGen failed", t);
+            }
+
+        }
+        log.info("End topic queue mapping clean service  thread!");
+    }
+
+
+
+    public void cleanItemExpired() {
+        String when = messageStoreConfig.getDeleteWhen();
+        if (!UtilAll.isItTimeToDo(when)) {
+            return;
+        }
+        boolean changed = false;
+        long start = System.currentTimeMillis();
+        try {
+            for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {
+                try {
+                    if (isStopped()) {
+                        break;
+                    }
+                    TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);
+                    if (mappingDetail == null
+                            || mappingDetail.getHostedQueues().isEmpty()) {
+                        continue;
+                    }
+                    if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {
+                        log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail);
+                        continue;
+                    }
+                    Set<String> brokers = new HashSet<>();
+                    for (List<LogicQueueMappingItem> items: mappingDetail.getHostedQueues().values()) {
+                        if (items.size() <= 1) {
+                            continue;
+                        }
+                        if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {
+                            continue;
+                        }
+                        LogicQueueMappingItem earlistItem = items.get(0);
+                        brokers.add(earlistItem.getBname());
+                    }
+                    Map<String, TopicStatsTable> statsTable = new HashMap<>();
+                    for (String broker: brokers) {
+                        GetTopicStatsInfoRequestHeader header = new GetTopicStatsInfoRequestHeader();
+                        header.setTopic(topic);
+                        header.setBname(broker);
+                        header.setLo(false);
+                        try {
+                            RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_STATS_INFO, header, null);
+                            RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();
+                            if (rpcResponse.getException() != null) {
+                                throw rpcResponse.getException();
+                            }
+                            statsTable.put(broker, (TopicStatsTable) rpcResponse.getBody());
+                        } catch (Throwable rt) {
+                            log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt);
+                        }
+                    }
+                    Map<Integer, List<LogicQueueMappingItem>> newHostedQueues = new HashMap<>();
+                    boolean changedForTopic = false;
+                    for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {
+                        Integer qid = entry.getKey();
+                        List<LogicQueueMappingItem> items = entry.getValue();
+                        if (items.size() <= 1) {
+                            continue;
+                        }
+                        if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {
+                            continue;
+                        }
+                        LogicQueueMappingItem earlistItem = items.get(0);
+                        TopicStatsTable topicStats = statsTable.get(earlistItem.getBname());
+                        if (topicStats == null) {
+                            continue;
+                        }
+                        TopicOffset topicOffset = topicStats.getOffsetTable().get(new MessageQueue(topic, earlistItem.getBname(), earlistItem.getQueueId()));
+                        if (topicOffset == null) {
+                            //this may should not happen
+                            log.error("Get null topicOffset for {} {}",topic,  earlistItem);
+                            continue;
+                        }
+                        //ignore the maxOffset < 0, which may in case of some error
+                        if (topicOffset.getMaxOffset() == topicOffset.getMinOffset()
+                            || topicOffset.getMaxOffset() == 0) {
+                            List<LogicQueueMappingItem> newItems = new ArrayList<>(items);
+                            boolean result = newItems.remove(earlistItem);
+                            if (result) {
+                                changedForTopic = true;
+                                newHostedQueues.put(qid, newItems);
+                            }
+                            log.info("The logic queue item {} {} is removed {} because of {}", topic, earlistItem, result, topicOffset);
+                        }
+                    }
+                    if (changedForTopic) {
+                        TopicQueueMappingDetail newMappingDetail = new TopicQueueMappingDetail(mappingDetail.getTopic(), mappingDetail.getTotalQueues(), mappingDetail.getBname(), mappingDetail.getEpoch());
+                        newMappingDetail.getHostedQueues().putAll(mappingDetail.getHostedQueues());
+                        newMappingDetail.getHostedQueues().putAll(newHostedQueues);
+                        this.topicQueueMappingManager.updateTopicQueueMapping(newMappingDetail, false, true, false);
+                        changed = true;
+                    }
+                } catch (Throwable tt) {
+                    log.error("Try CleanItemExpired failed for {}", topic, tt);
+                } finally {
+                    UtilAll.sleep(10);
+                }
+            }
+        } catch (Throwable t) {
+            log.error("Try cleanItemExpired failed", t);
+        } finally {
+            if (changed) {
+                this.topicQueueMappingManager.getDataVersion().nextVersion();
+                this.topicQueueMappingManager.persist();
+                log.info("CleanItemExpired changed");
+            }
+            log.info("cleanItemExpired cost {} ms", System.currentTimeMillis() - start);
+        }
+    }
+
+    public void cleanItemListMoreThanSecondGen() {
+        String when = messageStoreConfig.getDeleteWhen();
+        if (!UtilAll.isItTimeToDo(when)) {
+            return;
+        }
+        boolean changed = false;
+        long start = System.currentTimeMillis();
+        try {
+            ClientMetadata clientMetadata = new ClientMetadata();
+            for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {
+                try {
+                    if (isStopped()) {
+                        break;
+                    }
+                    TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);
+                    if (mappingDetail == null
+                            || mappingDetail.getHostedQueues().isEmpty()) {
+                        continue;
+                    }
+                    if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {
+                        log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail);
+                        continue;
+                    }
+                    Map<Integer, String> qid2CurrLeaderBroker = new HashMap<>();
+                    for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {
+                        Integer qId = entry.getKey();
+                        List<LogicQueueMappingItem> items = entry.getValue();
+                        if (items.isEmpty()) {
+                            continue;
+                        }
+                        LogicQueueMappingItem leaderItem = items.get(items.size() - 1);
+                        if (!leaderItem.getBname().equals(mappingDetail.getBname())) {
+                            qid2CurrLeaderBroker.put(qId, leaderItem.getBname());
+                        }
+                    }
+                    if (qid2CurrLeaderBroker.isEmpty()) {
+                        continue;
+                    }
+                    //find the topic route
+                    TopicRouteData topicRouteData = brokerOuterAPI.getTopicRouteInfoFromNameServer(topic, brokerConfig.getForwardTimeout());
+                    clientMetadata.freshTopicRoute(topic, topicRouteData);
+                    Map<Integer, String> qid2RealLeaderBroker = new HashMap<>();
+                    //fine the real leader
+                    for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {
+                        qid2RealLeaderBroker.put(entry.getKey(), clientMetadata.getBrokerNameFromMessageQueue(new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(mappingDetail.getScope()), entry.getKey())));
+                    }
+
+                    //find the mapping detail of real leader
+                    Map<String, TopicQueueMappingDetail> mappingDetailMap = new HashMap<>();
+                    for (Map.Entry<Integer, String> entry : qid2RealLeaderBroker.entrySet()) {
+                        if (entry.getValue().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {
+                            continue;
+                        }
+                        String broker = entry.getValue();
+                        GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader();
+                        header.setTopic(topic);
+                        header.setBname(broker);
+                        header.setLo(true);
+                        try {
+                            RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_CONFIG, header, null);
+                            RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();
+                            if (rpcResponse.getException() != null) {
+                                throw rpcResponse.getException();
+                            }
+                            TopicQueueMappingDetail mappingDetailRemote = ((TopicConfigAndQueueMapping) rpcResponse.getBody()).getMappingDetail();
+                            if (broker.equals(mappingDetailRemote.getBname())) {
+                                mappingDetailMap.put(broker, mappingDetailRemote);
+                            }
+                        } catch (Throwable rt) {
+                            log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt);
+                        }
+                    }
+                    //check all the info
+                    Set<Integer> ids2delete = new HashSet<>();
+                    for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {
+                        Integer qId = entry.getKey();
+                        String currLeaderBroker = entry.getValue();
+                        String realLeaderBroker = qid2RealLeaderBroker.get(qId);
+                        TopicQueueMappingDetail remoteMappingDetail = mappingDetailMap.get(realLeaderBroker);
+                        if (remoteMappingDetail == null
+                                || remoteMappingDetail.getTotalQueues() != mappingDetail.getTotalQueues()
+                                || remoteMappingDetail.getEpoch() != mappingDetail.getEpoch()) {
+                            continue;
+                        }
+                        List<LogicQueueMappingItem> items = remoteMappingDetail.getHostedQueues().get(qId);
+                        if (items.isEmpty()) {
+                            continue;
+                        }
+                        LogicQueueMappingItem leaderItem = items.get(items.size() - 1);
+                        if (!realLeaderBroker.equals(leaderItem.getBname())) {
+                            continue;
+                        }
+                        //all the check is ok
+                        if (!realLeaderBroker.equals(currLeaderBroker)) {
+                            ids2delete.add(qId);
+                        }
+                    }
+                    for (Integer qid : ids2delete) {
+                        List<LogicQueueMappingItem> items = mappingDetail.getHostedQueues().remove(qid);
+                        changed =  true;
+                        if (items != null) {
+                            log.info("Remove the ItemListMoreThanSecondGen topic {} qid {} items {}", topic, qid, items);
+                        }
+                    }
+                } catch (Throwable tt) {
+                    log.error("Try cleanItemListMoreThanSecondGen failed for topic {}", topic, tt);
+                } finally {
+                    UtilAll.sleep(10);
+                }
+            }
+        } catch (Throwable t) {
+            log.error("Try cleanItemListMoreThanSecondGen failed", t);
+        } finally {
+            if (changed) {
+                this.topicQueueMappingManager.getDataVersion().nextVersion();
+                this.topicQueueMappingManager.persist();
+            }
+            log.info("Try cleanItemListMoreThanSecondGen cost {} ms", System.currentTimeMillis() - start);
+        }
+    }
+
+
+
+
+}

broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java

@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.topic;
+
+import com.alibaba.fastjson.JSON;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.BrokerPathConfigHelper;
+import org.apache.rocketmq.common.ConfigManager;
+import org.apache.rocketmq.common.DataVersion;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.TopicQueueMappingSerializeWrapper;
+import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
+import org.apache.rocketmq.common.rpc.TopicRequestHeader;
+import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
+import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
+
+public class TopicQueueMappingManager extends ConfigManager {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private static final long LOCK_TIMEOUT_MILLIS = 3000;
+    private transient final Lock lock = new ReentrantLock();
+
+    //this data version should be equal to the TopicConfigManager
+    private final DataVersion dataVersion = new DataVersion();
+    private transient BrokerController brokerController;
+
+    private final ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();
+
+
+    public TopicQueueMappingManager(BrokerController brokerController) {
+        this.brokerController = brokerController;
+    }
+
+    public void updateTopicQueueMapping(TopicQueueMappingDetail newDetail, boolean force, boolean isClean, boolean flush) throws Exception {
+        boolean locked = false;
+        boolean updated = false;
+        TopicQueueMappingDetail oldDetail = null;
+        try {
+
+            if (lock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
+                locked = true;
+            } else {
+                return;
+            }
+            if (newDetail == null) {
+                return;
+            }
+            assert newDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());
+
+            newDetail.getHostedQueues().forEach((queueId, items) -> {
+                TopicQueueMappingUtils.checkLogicQueueMappingItemOffset(items);
+            });
+
+            oldDetail = topicQueueMappingTable.get(newDetail.getTopic());
+            if (oldDetail == null) {
+                topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
+                updated = true;
+                return;
+            }
+            if (force) {
+                //bakeup the old items
+                oldDetail.getHostedQueues().forEach((queueId, items) -> {
+                    newDetail.getHostedQueues().putIfAbsent(queueId, items);
+                });
+                topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
+                updated = true;
+                return;
+            }
+            //do more check
+            if (newDetail.getEpoch() < oldDetail.getEpoch()) {
+                throw new RuntimeException(String.format("Can't accept data with small epoch %d < %d", newDetail.getEpoch(), oldDetail.getEpoch()));
+            }
+            if (!newDetail.getScope().equals(oldDetail.getScope())) {
+                throw new RuntimeException(String.format("Can't accept data with unmatched scope %s != %s", newDetail.getScope(), oldDetail.getScope()));
+            }
+            boolean epochEqual = newDetail.getEpoch() == oldDetail.getEpoch();
+            for (Integer globalId : oldDetail.getHostedQueues().keySet()) {
+                List<LogicQueueMappingItem> oldItems = oldDetail.getHostedQueues().get(globalId);
+                List<LogicQueueMappingItem> newItems = newDetail.getHostedQueues().get(globalId);
+                if (newItems == null) {
+                    if (epochEqual) {
+                        throw new RuntimeException("Cannot accept equal epoch with null data");
+                    } else {
+                        newDetail.getHostedQueues().put(globalId, oldItems);
+                    }
+                } else {
+                    TopicQueueMappingUtils.makeSureLogicQueueMappingItemImmutable(oldItems, newItems, epochEqual, isClean);
+                }
+            }
+            topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
+            updated = true;
+        }  finally {
+            if (locked) {
+                this.lock.unlock();
+            }
+            if (updated && flush) {
+                this.dataVersion.nextVersion();
+                this.persist();
+                log.info("Update topic queue mapping from [{}] to [{}], force {}", oldDetail, newDetail, force);
+            }
+        }
+
+    }
+
+    public void delete(final String topic) {
+        TopicQueueMappingDetail old = this.topicQueueMappingTable.remove(topic);
+        if (old != null) {
+            log.info("delete topic queue mapping OK, static topic queue mapping: {}", old);
+            this.dataVersion.nextVersion();
+            this.persist();
+        } else {
+            log.warn("delete topic queue mapping failed, static topic: {} not exists", topic);
+        }
+    }
+
+    public TopicQueueMappingDetail getTopicQueueMapping(String topic) {
+        return topicQueueMappingTable.get(topic);
+    }
+
+    @Override
+    public String encode(boolean pretty) {
+        TopicQueueMappingSerializeWrapper wrapper = new TopicQueueMappingSerializeWrapper();
+        wrapper.setTopicQueueMappingInfoMap(topicQueueMappingTable);
+        wrapper.setDataVersion(this.dataVersion);
+        return JSON.toJSONString(wrapper, pretty);
+    }
+
+    @Override
+    public String encode() {
+        return encode(false);
+    }
+
+    @Override
+    public String configFilePath() {
+        return BrokerPathConfigHelper.getTopicQueueMappingPath(this.brokerController.getMessageStoreConfig()
+            .getStorePathRootDir());
+    }
+
+    @Override
+    public void decode(String jsonString) {
+        if (jsonString != null) {
+            TopicQueueMappingSerializeWrapper wrapper = TopicQueueMappingSerializeWrapper.fromJson(jsonString, TopicQueueMappingSerializeWrapper.class);
+            if (wrapper != null) {
+                this.topicQueueMappingTable.putAll(wrapper.getTopicQueueMappingInfoMap());
+                this.dataVersion.assignNewOne(wrapper.getDataVersion());
+            }
+        }
+    }
+
+    public ConcurrentMap<String, TopicQueueMappingDetail> getTopicQueueMappingTable() {
+        return topicQueueMappingTable;
+    }
+
+    public DataVersion getDataVersion() {
+        return dataVersion;
+    }
+
+    public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader) {
+        return buildTopicQueueMappingContext(requestHeader, false);
+    }
+
+    //Do not return a null context
+    public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader, boolean selectOneWhenMiss) {
+        // if lo is set to false explicitly, it maybe the forwarded request
+        if (requestHeader.getLo() != null
+                && Boolean.FALSE.equals(requestHeader.getLo())) {
+            return new TopicQueueMappingContext(requestHeader.getTopic(), null, null, null, null);
+        }
+        String topic = requestHeader.getTopic();
+        Integer globalId = null;
+        if (requestHeader instanceof  TopicQueueRequestHeader) {
+            globalId = ((TopicQueueRequestHeader) requestHeader).getQueueId();
+        }
+
+        TopicQueueMappingDetail mappingDetail = getTopicQueueMapping(topic);
+        if (mappingDetail == null) {
+            //it is not static topic
+            return new TopicQueueMappingContext(topic, null, null, null, null);
+        }
+        assert mappingDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());
+
+        if (globalId == null) {
+            return new TopicQueueMappingContext(topic, null, mappingDetail, null, null);
+        }
+
+        //If not find mappingItem, it encounters some errors
+        if (globalId < 0 && !selectOneWhenMiss) {
+            return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null);
+        }
+
+        if (globalId < 0) {
+            try {
+                if (!mappingDetail.getHostedQueues().isEmpty()) {
+                    //do not check
+                    globalId = mappingDetail.getHostedQueues().keySet().iterator().next();
+                }
+            } catch (Throwable ignored) {
+            }
+        }
+        if (globalId < 0) {
+            return new TopicQueueMappingContext(topic, globalId,  mappingDetail, null, null);
+        }
+
+        List<LogicQueueMappingItem> mappingItemList = TopicQueueMappingDetail.getMappingInfo(mappingDetail, globalId);
+        LogicQueueMappingItem leaderItem = null;
+        if (mappingItemList != null
+                && mappingItemList.size() > 0) {
+            leaderItem = mappingItemList.get(mappingItemList.size() - 1);
+        }
+        return new TopicQueueMappingContext(topic, globalId, mappingDetail, mappingItemList, leaderItem);
+    }
+
+
+    public  RemotingCommand rewriteRequestForStaticTopic(TopicQueueRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {
+        try {
+            if (mappingContext.getMappingDetail() == null) {
+                return null;
+            }
+            TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
+            if (!mappingContext.isLeader()) {
+                return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));
+            }
+            LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();
+            requestHeader.setQueueId(mappingItem.getQueueId());
+            return null;
+        } catch (Throwable t) {
+            return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
+        }
+    }
+
+}

broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java

@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.transaction;
 
 import io.netty.channel.Channel;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -27,7 +28,6 @@ import org.apache.rocketmq.logging.InternalLoggerFactory;
 
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;
 import java.util.concurrent.TimeUnit;
@@ -40,14 +40,7 @@ public abstract class AbstractTransactionalMessageCheckListener {
     //queue nums of topic TRANS_CHECK_MAX_TIME_TOPIC
     protected final static int TCMT_QUEUE_NUMS = 1;
 
-    private static ExecutorService executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000), new ThreadFactory() {
-        @Override
-        public Thread newThread(Runnable r) {
-            Thread thread = new Thread(r);
-            thread.setName("Transaction-msg-check-thread");
-            return thread;
-        }
-    }, new CallerRunsPolicy());
+    private static volatile ExecutorService executorService;
 
     public AbstractTransactionalMessageCheckListener() {
     }
@@ -76,16 +69,20 @@ public abstract class AbstractTransactionalMessageCheckListener {
     }
 
     public void resolveHalfMsg(final MessageExt msgExt) {
-        executorService.execute(new Runnable() {
-            @Override
-            public void run() {
-                try {
-                    sendCheckMessage(msgExt);
-                } catch (Exception e) {
-                    LOGGER.error("Send check message error!", e);
+        if (executorService != null) {
+            executorService.execute(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        sendCheckMessage(msgExt);
+                    } catch (Exception e) {
+                        LOGGER.error("Send check message error!", e);
+                    }
                 }
-            }
-        });
+            });
+        } else {
+            LOGGER.error("TransactionalMessageCheckListener not init");
+        }
     }
 
     public BrokerController getBrokerController() {
@@ -93,7 +90,16 @@ public abstract class AbstractTransactionalMessageCheckListener {
     }
 
     public void shutDown() {
-        executorService.shutdown();
+        if (executorService != null) {
+            executorService.shutdown();
+        }
+    }
+
+    public synchronized void initExecutorService() {
+        if (executorService == null) {
+            executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(2000),
+                new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy());
+        }
     }
 
     /**
@@ -103,6 +109,7 @@ public abstract class AbstractTransactionalMessageCheckListener {
      */
     public void setBrokerController(BrokerController brokerController) {
         this.brokerController = brokerController;
+        initExecutorService();
     }
 
     /**

broker/src/main/java/org/apache/rocketmq/broker/transaction/OperationResult.java

@@ -25,10 +25,6 @@ public class OperationResult {
 
     private String responseRemark;
 
-    public MessageExt getPrepareMessage() {
-        return prepareMessage;
-    }
-
     public void setPrepareMessage(MessageExt prepareMessage) {
         this.prepareMessage = prepareMessage;
     }
@@ -48,4 +44,8 @@ public class OperationResult {
     public void setResponseRemark(String responseRemark) {
         this.responseRemark = responseRemark;
     }
+
+    public MessageExt getPrepareMessage() {
+        return prepareMessage;
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java

@@ -33,6 +33,9 @@ public class TransactionalMessageCheckService extends ServiceThread {
 
     @Override
     public String getServiceName() {
+        if (brokerController != null && brokerController.getBrokerConfig().isInBrokerContainer()) {
+            return brokerController.getBrokerIdentity().getLoggerIdentifier() + TransactionalMessageCheckService.class.getSimpleName();
+        }
         return TransactionalMessageCheckService.class.getSimpleName();
     }
 

broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageService.java

@@ -18,7 +18,7 @@ package org.apache.rocketmq.broker.transaction;
 
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import java.util.concurrent.CompletableFuture;
 

broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java

@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.broker.transaction.jdbc;
-
-import java.net.URL;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
-import java.util.Properties;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.rocketmq.broker.transaction.TransactionRecord;
-import org.apache.rocketmq.broker.transaction.TransactionStore;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
-
-public class JDBCTransactionStore implements TransactionStore {
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
-    private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig;
-    private Connection connection;
-    private AtomicLong totalRecordsValue = new AtomicLong(0);
-
-    public JDBCTransactionStore(JDBCTransactionStoreConfig jdbcTransactionStoreConfig) {
-        this.jdbcTransactionStoreConfig = jdbcTransactionStoreConfig;
-    }
-
-    @Override
-    public boolean open() {
-        if (this.loadDriver()) {
-            Properties props = new Properties();
-            props.put("user", jdbcTransactionStoreConfig.getJdbcUser());
-            props.put("password", jdbcTransactionStoreConfig.getJdbcPassword());
-
-            try {
-                this.connection =
-                    DriverManager.getConnection(this.jdbcTransactionStoreConfig.getJdbcURL(), props);
-
-                this.connection.setAutoCommit(false);
-
-                if (!this.computeTotalRecords()) {
-                    return this.createDB();
-                }
-
-                return true;
-            } catch (SQLException e) {
-                log.info("Create JDBC Connection Exception", e);
-            }
-        }
-
-        return false;
-    }
-
-    private boolean loadDriver() {
-        try {
-            Class.forName(this.jdbcTransactionStoreConfig.getJdbcDriverClass()).getDeclaredConstructor().newInstance();
-            log.info("Loaded the appropriate driver, {}",
-                this.jdbcTransactionStoreConfig.getJdbcDriverClass());
-            return true;
-        } catch (Exception e) {
-            log.info("Loaded the appropriate driver Exception", e);
-        }
-
-        return false;
-    }
-
-    private boolean computeTotalRecords() {
-        Statement statement = null;
-        ResultSet resultSet = null;
-        try {
-            statement = this.connection.createStatement();
-
-            resultSet = statement.executeQuery("select count(offset) as total from t_transaction");
-            if (!resultSet.next()) {
-                log.warn("computeTotalRecords ResultSet is empty");
-                return false;
-            }
-
-            this.totalRecordsValue.set(resultSet.getLong(1));
-        } catch (Exception e) {
-            log.warn("computeTotalRecords Exception", e);
-            return false;
-        } finally {
-            if (null != statement) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                }
-            }
-
-            if (null != resultSet) {
-                try {
-                    resultSet.close();
-                } catch (SQLException e) {
-                }
-            }
-        }
-
-        return true;
-    }
-
-    private boolean createDB() {
-        Statement statement = null;
-        try {
-            statement = this.connection.createStatement();
-
-            String sql = this.createTableSql();
-            log.info("createDB SQL:\n {}", sql);
-            statement.execute(sql);
-            this.connection.commit();
-            return true;
-        } catch (Exception e) {
-            log.warn("createDB Exception", e);
-            return false;
-        } finally {
-            if (null != statement) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                    log.warn("Close statement exception", e);
-                }
-            }
-        }
-    }
-
-    private String createTableSql() {
-        URL resource = JDBCTransactionStore.class.getClassLoader().getResource("transaction.sql");
-        String fileContent = MixAll.file2String(resource);
-        return fileContent;
-    }
-
-    @Override
-    public void close() {
-        try {
-            if (this.connection != null) {
-                this.connection.close();
-            }
-        } catch (SQLException e) {
-        }
-    }
-
-    @Override
-    public boolean put(List<TransactionRecord> trs) {
-        PreparedStatement statement = null;
-        try {
-            this.connection.setAutoCommit(false);
-            statement = this.connection.prepareStatement("insert into t_transaction values (?, ?)");
-            for (TransactionRecord tr : trs) {
-                statement.setLong(1, tr.getOffset());
-                statement.setString(2, tr.getProducerGroup());
-                statement.addBatch();
-            }
-            int[] executeBatch = statement.executeBatch();
-            this.connection.commit();
-            this.totalRecordsValue.addAndGet(updatedRows(executeBatch));
-            return true;
-        } catch (Exception e) {
-            log.warn("createDB Exception", e);
-            return false;
-        } finally {
-            if (null != statement) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                    log.warn("Close statement exception", e);
-                }
-            }
-        }
-    }
-
-    private long updatedRows(int[] rows) {
-        long res = 0;
-        for (int i : rows) {
-            res += i;
-        }
-
-        return res;
-    }
-
-    @Override
-    public void remove(List<Long> pks) {
-        PreparedStatement statement = null;
-        try {
-            this.connection.setAutoCommit(false);
-            statement = this.connection.prepareStatement("DELETE FROM t_transaction WHERE offset = ?");
-            for (long pk : pks) {
-                statement.setLong(1, pk);
-                statement.addBatch();
-            }
-            int[] executeBatch = statement.executeBatch();
-            this.connection.commit();
-        } catch (Exception e) {
-            log.warn("createDB Exception", e);
-        } finally {
-            if (null != statement) {
-                try {
-                    statement.close();
-                } catch (SQLException e) {
-                }
-            }
-        }
-    }
-
-    @Override
-    public List<TransactionRecord> traverse(long pk, int nums) {
-        return null;
-    }
-
-    @Override
-    public long totalRecords() {
-        return this.totalRecordsValue.get();
-    }
-
-    @Override
-    public long minPK() {
-        return 0;
-    }
-
-    @Override
-    public long maxPK() {
-        return 0;
-    }
-}

broker/src/main/java/org/apache/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java

@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.broker.transaction.jdbc;
-
-public class JDBCTransactionStoreConfig {
-    private String jdbcDriverClass = "com.mysql.jdbc.Driver";
-    private String jdbcURL = "jdbc:mysql://xxx.xxx.xxx.xxx:1000/xxx?useUnicode=true&characterEncoding=UTF-8";
-    private String jdbcUser = "xxx";
-    private String jdbcPassword = "xxx";
-
-    public String getJdbcDriverClass() {
-        return jdbcDriverClass;
-    }
-
-    public void setJdbcDriverClass(String jdbcDriverClass) {
-        this.jdbcDriverClass = jdbcDriverClass;
-    }
-
-    public String getJdbcURL() {
-        return jdbcURL;
-    }
-
-    public void setJdbcURL(String jdbcURL) {
-        this.jdbcURL = jdbcURL;
-    }
-
-    public String getJdbcUser() {
-        return jdbcUser;
-    }
-
-    public void setJdbcUser(String jdbcUser) {
-        this.jdbcUser = jdbcUser;
-    }
-
-    public String getJdbcPassword() {
-        return jdbcPassword;
-    }
-
-    public void setJdbcPassword(String jdbcPassword) {
-        this.jdbcPassword = jdbcPassword;
-    }
-}

broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/DefaultTransactionalMessageCheckListener.java

@@ -26,7 +26,7 @@ import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 

broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java

@@ -35,7 +35,7 @@ import org.apache.rocketmq.logging.InnerLoggerFactory;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.store.GetMessageResult;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;

broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java

@@ -30,7 +30,7 @@ import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
 import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 

broker/src/main/java/org/apache/rocketmq/broker/util/HookUtils.java

@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.util;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.common.utils.QueueTypeUtils;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.config.BrokerRole;
+
+public class HookUtils {
+
+    protected static final InternalLogger LOG = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+
+    private static AtomicLong printTimes = new AtomicLong(0);
+
+    public static PutMessageResult checkBeforePutMessage(BrokerController brokerController, final MessageExt msg) {
+        if (brokerController.getMessageStore().isShutdown()) {
+            LOG.warn("message store has shutdown, so putMessage is forbidden");
+            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+        }
+
+        if (!brokerController.getMessageStoreConfig().isDuplicationEnable() && BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) {
+            long value = printTimes.getAndIncrement();
+            if ((value % 50000) == 0) {
+                LOG.warn("message store is in slave mode, so putMessage is forbidden ");
+            }
+
+            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+        }
+
+        if (!brokerController.getMessageStore().getRunningFlags().isWriteable()) {
+            long value = printTimes.getAndIncrement();
+            if ((value % 50000) == 0) {
+                LOG.warn("message store is not writeable, so putMessage is forbidden " + brokerController.getMessageStore().getRunningFlags().getFlagBits());
+            }
+
+            return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null);
+        } else {
+            printTimes.set(0);
+        }
+
+        final byte[] topicData = msg.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);
+        final int topicLength = topicData == null ? 0 : topicData.length;
+
+        if (topicLength > brokerController.getMessageStoreConfig().getMaxTopicLength()) {
+            LOG.warn("putMessage message topic[{}] length too long {}", msg.getTopic(), topicLength);
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+        }
+
+        if (topicLength > Byte.MAX_VALUE) {
+            LOG.warn("putMessage message topic[{}] length too long {}, but it is not supported by broker",
+                msg.getTopic(), topicLength);
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+        }
+
+        if (msg.getBody() == null) {
+            LOG.warn("putMessage message topic[{}], but message body is null", msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+        }
+
+        if (msg.getBody() == null) {
+            LOG.warn("putMessage message topic[{}], but message body is null", msg.getTopic());
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+        }
+
+        if (brokerController.getMessageStore().isOSPageCacheBusy()) {
+            return new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, null);
+        }
+        return null;
+    }
+
+    public static PutMessageResult checkInnerBatch(BrokerController brokerController, final MessageExt msg) {
+        if (msg.getProperties().containsKey(MessageConst.PROPERTY_INNER_NUM)
+            && !MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+            LOG.warn("[BUG]The message had property {} but is not an inner batch", MessageConst.PROPERTY_INNER_NUM);
+            return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+        }
+
+        if (MessageSysFlag.check(msg.getSysFlag(), MessageSysFlag.INNER_BATCH_FLAG)) {
+            Optional<TopicConfig> topicConfig = Optional.ofNullable(brokerController.getTopicConfigManager().getTopicConfigTable().get(msg.getTopic()));
+            if (!QueueTypeUtils.isBatchCq(topicConfig)) {
+                LOG.error("[BUG]The message is an inner batch but cq type is not batch cq");
+                return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+            }
+        }
+
+        return null;
+    }
+
+    public static PutMessageResult handleScheduleMessage(BrokerController brokerController,
+        final MessageExtBrokerInner msg) {
+        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
+        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE
+            || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {
+            if (!TopicValidator.RMQ_SYS_SCHEDULE_TOPIC.equals(msg.getTopic()) && msg.getDelayTimeLevel() > 0) {
+                transformDelayLevelMessage(brokerController, msg);
+            }
+        }
+        return null;
+    }
+
+    public static void transformDelayLevelMessage(BrokerController brokerController, MessageExtBrokerInner msg) {
+
+        if (msg.getDelayTimeLevel() > brokerController.getScheduleMessageService().getMaxDelayLevel()) {
+            msg.setDelayTimeLevel(brokerController.getScheduleMessageService().getMaxDelayLevel());
+        }
+
+        // Backup real topic, queueId
+        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());
+        MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));
+        msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+
+        msg.setTopic(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);
+        msg.setQueueId(ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()));
+    }
+
+    public static boolean sendMessageBack(BrokerController brokerController, List<MessageExt> msgList,
+        String brokerName, String brokerAddr) {
+        try {
+            Iterator<MessageExt> it = msgList.iterator();
+            while (it.hasNext()) {
+                MessageExt msg = it.next();
+                msg.setWaitStoreMsgOK(false);
+                brokerController.getBrokerOuterAPI().sendMessageToSpecificBroker(brokerAddr, brokerName, msg, "InnerSendMessageBackGroup", 3000);
+                it.remove();
+            }
+        } catch (Exception e) {
+            LOG.error("send message back to broker {} addr {} failed", brokerName, brokerAddr, e);
+            return false;
+        }
+        return true;
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/util/MsgUtil.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.util;
+
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageExt;
+
+public final class MsgUtil {
+    private MsgUtil() {
+    }
+
+    public static void setMessageDeliverTime(BrokerController brokerController, Message msgInner, long timeMillis) {
+        msgInner.setDelayTimeLevel(brokerController.getScheduleMessageService().computeDelayLevel(timeMillis));
+    }
+
+    public static long getMessageDeliverTime(BrokerController brokerController, MessageExt msgInner) {
+        return brokerController.getScheduleMessageService().computeDeliverTimestamp(msgInner.getDelayTimeLevel(), msgInner.getStoreTimestamp());
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java

@@ -30,7 +30,6 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.netty.RequestTask;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.After;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -44,7 +43,7 @@ public class BrokerControllerTest {
             new NettyServerConfig(),
             new NettyClientConfig(),
             new MessageStoreConfig());
-        assertThat(brokerController.initialize());
+        assertThat(brokerController.initialize()).isTrue();
         brokerController.start();
         brokerController.shutdown();
     }

broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java

@@ -22,19 +22,27 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import io.netty.channel.ChannelHandlerContext;
 import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.broker.out.BrokerOuterAPI;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.BrokerIdentity;
+import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
+import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.ClusterInfo;
 import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
 import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader;
 import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.junit.Test;
@@ -50,6 +58,8 @@ import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.isNull;
 import static org.mockito.Mockito.when;
 
 @RunWith(MockitoJUnitRunner.class)
@@ -75,7 +85,7 @@ public class BrokerOuterAPITest {
     private BrokerOuterAPI brokerOuterAPI;
 
     public void init() throws Exception {
-        brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), null);
+        brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig());
         Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
         field.setAccessible(true);
         field.set(brokerOuterAPI, nettyRemotingClient);
@@ -91,7 +101,7 @@ public class BrokerOuterAPITest {
 
         when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
         when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response);
-        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut);
+        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut, false);
         assertTrue(booleanList.size() > 0);
         assertEquals(false, booleanList.contains(Boolean.FALSE));
     }
@@ -119,7 +129,7 @@ public class BrokerOuterAPITest {
                 return buildResponse(Boolean.TRUE);
             }
         });
-        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut);
+        List<Boolean> booleanList = brokerOuterAPI.needRegister(clusterName, brokerAddr, brokerName, brokerId, topicConfigSerializeWrapper, timeOut, false);
         assertEquals(2, booleanList.size());
         boolean success = Iterables.any(booleanList,
             new Predicate<Boolean>() {
@@ -144,9 +154,25 @@ public class BrokerOuterAPITest {
 
         TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
 
-        when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
-        when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenReturn(response);
-        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, true);
+        when(nettyRemotingClient.getAvailableNameSrvList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
+        when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).then(new Answer<RemotingCommand>() {
+            @Override
+            public RemotingCommand answer(InvocationOnMock mock) throws Throwable {
+                RemotingCommand request = mock.getArgument(1);
+                return response;
+            }
+        });
+        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr,
+            brokerName,
+            brokerId,
+            "hasServerAddr",
+            topicConfigSerializeWrapper,
+            Lists.newArrayList(),
+            false,
+            timeOut,
+            false,
+            true,
+            new BrokerIdentity());
 
         assertTrue(registerBrokerResultList.size() > 0);
     }
@@ -162,7 +188,7 @@ public class BrokerOuterAPITest {
 
         TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
 
-        when(nettyRemotingClient.getNameServerAddressList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
+        when(nettyRemotingClient.getAvailableNameSrvList()).thenReturn(Lists.asList(nameserver1, nameserver2, new String[] {nameserver3}));
         when(nettyRemotingClient.invokeSync(anyString(), any(RemotingCommand.class), anyLong())).thenAnswer(new Answer<RemotingCommand>() {
             @Override
             public RemotingCommand answer(InvocationOnMock invocation) throws Throwable {
@@ -177,11 +203,30 @@ public class BrokerOuterAPITest {
                 return response;
             }
         });
-        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, true);
+        List<RegisterBrokerResult> registerBrokerResultList = brokerOuterAPI.registerBrokerAll(clusterName, brokerAddr, brokerName, brokerId, "hasServerAddr", topicConfigSerializeWrapper, Lists.<String>newArrayList(), false, timeOut, false, true, new BrokerIdentity());
 
         assertEquals(2, registerBrokerResultList.size());
     }
 
+    @Test
+    public void testGetBrokerClusterInfo() throws Exception {
+        init();
+        brokerOuterAPI.start();
+
+        final RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(ResponseCode.SUCCESS);
+        response.setRemark(null);
+
+        ClusterInfo want = new ClusterInfo();
+        want.setBrokerAddrTable(new HashMap<>(Collections.singletonMap("key", new BrokerData("cluster", "broker", new HashMap<>(Collections.singletonMap(MixAll.MASTER_ID, "127.0.0.1:10911"))))));
+        response.setBody(RemotingSerializable.encode(want));
+
+        when(nettyRemotingClient.invokeSync(isNull(), argThat(argument -> argument.getCode() == RequestCode.GET_BROKER_CLUSTER_INFO), anyLong())).thenReturn(response);
+        ClusterInfo got = brokerOuterAPI.getBrokerClusterInfo();
+
+        assertEquals(want, got);
+    }
+
     private RemotingCommand buildResponse(Boolean changed) {
         final RemotingCommand response = RemotingCommand.createResponseCommand(QueryDataVersionResponseHeader.class);
         final QueryDataVersionResponseHeader responseHeader = (QueryDataVersionResponseHeader) response.readCustomHeader();

broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java

@@ -20,6 +20,7 @@ package org.apache.rocketmq.broker;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.Properties;
+import org.apache.rocketmq.common.MixAll;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -34,8 +35,21 @@ public class BrokerStartupTest {
         Class<BrokerStartup> clazz = BrokerStartup.class;
         Method method = clazz.getDeclaredMethod("properties2SystemEnv", Properties.class);
         method.setAccessible(true);
-        System.setProperty("rocketmq.namesrv.domain", "value");
-        method.invoke(null, properties);
-        Assert.assertEquals("value", System.getProperty("rocketmq.namesrv.domain"));
+        {
+            properties.put("rmqAddressServerDomain", "value1");
+            properties.put("rmqAddressServerSubGroup", "value2");
+            method.invoke(null, properties);
+            Assert.assertEquals("value1", System.getProperty("rocketmq.namesrv.domain"));
+            Assert.assertEquals("value2", System.getProperty("rocketmq.namesrv.domain.subgroup"));
+        }
+        {
+            properties.put("rmqAddressServerDomain", MixAll.WS_DOMAIN_NAME);
+            properties.put("rmqAddressServerSubGroup", MixAll.WS_DOMAIN_SUBGROUP);
+            method.invoke(null, properties);
+            Assert.assertEquals(MixAll.WS_DOMAIN_NAME, System.getProperty("rocketmq.namesrv.domain"));
+            Assert.assertEquals(MixAll.WS_DOMAIN_SUBGROUP, System.getProperty("rocketmq.namesrv.domain.subgroup"));
+        }
+
+
     }
 }

broker/src/test/java/org/apache/rocketmq/broker/filter/MessageStoreWithFilterTest.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.broker.filter;
 
+import java.util.concurrent.TimeUnit;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.filter.ExpressionType;
@@ -30,11 +31,12 @@ import org.apache.rocketmq.store.DispatchRequest;
 import org.apache.rocketmq.store.GetMessageResult;
 import org.apache.rocketmq.store.GetMessageStatus;
 import org.apache.rocketmq.store.MessageArrivingListener;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageFilter;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.awaitility.core.ThrowingRunnable;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,6 +52,7 @@ import java.util.List;
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 
 public class MessageStoreWithFilterTest {
 
@@ -94,8 +97,10 @@ public class MessageStoreWithFilterTest {
 
     @After
     public void destroy() {
-        master.shutdown();
-        master.destroy();
+        if (master != null) {
+            master.shutdown();
+            master.destroy();
+        }
         UtilAll.deleteFile(new File(storePath));
     }
 
@@ -347,27 +352,30 @@ public class MessageStoreWithFilterTest {
     public void testGetMessage_withFilter_checkTagsCode() throws Exception {
         putMsg(master, topicCount, msgPerTopic);
 
-        Thread.sleep(200);
+        await().atMost(3, TimeUnit.SECONDS).untilAsserted(new ThrowingRunnable() {
+            @Override public void run() throws Throwable {
+                for (int i = 0; i < topicCount; i++) {
+                    final String realTopic = topic + i;
+                    GetMessageResult getMessageResult = master.getMessage("test", realTopic, queueId, 0, 10000,
+                        new MessageFilter() {
+                            @Override
+                            public boolean isMatchedByConsumeQueue(Long tagsCode,
+                                ConsumeQueueExt.CqExtUnit cqExtUnit) {
+                                if (tagsCode != null && tagsCode <= ConsumeQueueExt.MAX_ADDR) {
+                                    return false;
+                                }
+                                return true;
+                            }
 
-        for (int i = 0; i < topicCount; i++) {
-            String realTopic = topic + i;
-
-            GetMessageResult getMessageResult = master.getMessage("test", realTopic, queueId, 0, 10000,
-                new MessageFilter() {
-                    @Override
-                    public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
-                        if (tagsCode != null && tagsCode <= ConsumeQueueExt.MAX_ADDR) {
-                            return false;
-                        }
-                        return true;
-                    }
-
-                    @Override
-                    public boolean isMatchedByCommitLog(ByteBuffer msgBuffer, Map<String, String> properties) {
-                        return true;
-                    }
-                });
-            assertThat(getMessageResult.getMessageCount()).isEqualTo(msgPerTopic);
-        }
+                            @Override
+                            public boolean isMatchedByCommitLog(ByteBuffer msgBuffer,
+                                Map<String, String> properties) {
+                                return true;
+                            }
+                        });
+                    assertThat(getMessageResult.getMessageCount()).isEqualTo(msgPerTopic);
+                }
+            }
+        });
     }
 }

broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOffsetManagerTest.java

@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.broker.offset;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConsumerOffsetManagerTest {
+
+    private ConsumerOffsetManager consumerOffsetManager;
+
+    private static final String key = "FooBar@FooBarGroup";
+    @Before
+    public void init(){
+        consumerOffsetManager = new ConsumerOffsetManager();
+        ConcurrentHashMap<String, ConcurrentMap<Integer, Long>> offsetTable = new ConcurrentHashMap<String, ConcurrentMap<Integer, Long>>(512);
+        offsetTable.put(key,new ConcurrentHashMap<Integer, Long>(){{
+            put(1,2L);
+            put(2,3L);
+        }});
+        consumerOffsetManager.setOffsetTable(offsetTable);
+    }
+
+    @Test
+    public void cleanOffsetByTopic_NotExist(){
+        consumerOffsetManager.cleanOffsetByTopic("InvalidTopic");
+        assertThat(consumerOffsetManager.getOffsetTable().containsKey(key));
+    }
+
+    @Test
+    public void cleanOffsetByTopic_Exist(){
+        consumerOffsetManager.cleanOffsetByTopic("FooBar");
+        assertThat(!consumerOffsetManager.getOffsetTable().containsKey(key));
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/pagecache/OneMessageTransferTest.java

@@ -18,8 +18,8 @@
 package org.apache.rocketmq.broker.pagecache;
 
 import java.nio.ByteBuffer;
-import org.apache.rocketmq.store.MappedFile;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -29,7 +29,7 @@ public class OneMessageTransferTest {
     public void OneMessageTransferTest(){
         ByteBuffer byteBuffer = ByteBuffer.allocate(20);
         byteBuffer.putInt(20);
-        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());
         OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
     }
 
@@ -37,7 +37,7 @@ public class OneMessageTransferTest {
     public void OneMessageTransferCountTest(){
         ByteBuffer byteBuffer = ByteBuffer.allocate(20);
         byteBuffer.putInt(20);
-        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());
         OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
         Assert.assertEquals(manyMessageTransfer.count(),40);
     }
@@ -46,7 +46,7 @@ public class OneMessageTransferTest {
     public void OneMessageTransferPosTest(){
         ByteBuffer byteBuffer = ByteBuffer.allocate(20);
         byteBuffer.putInt(20);
-        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new MappedFile());
+        SelectMappedBufferResult selectMappedBufferResult = new SelectMappedBufferResult(0,byteBuffer,20,new DefaultMappedFile());
         OneMessageTransfer manyMessageTransfer = new OneMessageTransfer(byteBuffer,selectMappedBufferResult);
         Assert.assertEquals(manyMessageTransfer.position(),8);
     }

broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.lang.reflect.Field;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.net.Broker2Client;
+import org.apache.rocketmq.broker.failover.EscapeBridge;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AckMessageProcessorTest {
+    private AckMessageProcessor ackMessageProcessor;
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private DefaultMessageStore messageStore;
+    @Mock
+    private Channel channel;
+
+    private String topic = "FooBar";
+    private String group = "FooBarGroup";
+    private ClientChannelInfo clientInfo;
+    @Mock
+    private Broker2Client broker2Client;
+
+    @Before
+    public void init() throws IllegalAccessException, NoSuchFieldException {
+        clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0);
+        brokerController.setMessageStore(messageStore);
+        Field field = BrokerController.class.getDeclaredField("broker2Client");
+        field.setAccessible(true);
+        field.set(brokerController, broker2Client);
+        ScheduleMessageService scheduleMessageService = new ScheduleMessageService(brokerController);
+        scheduleMessageService.parseDelayLevel();
+        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);
+        EscapeBridge escapeBridge = new EscapeBridge(brokerController);
+        Mockito.when(brokerController.getEscapeBridge()).thenReturn(escapeBridge);
+        Channel mockChannel = mock(Channel.class);
+        when(handlerContext.channel()).thenReturn(mockChannel);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
+        ConsumerData consumerData = createConsumerData(group, topic);
+        brokerController.getConsumerManager().registerConsumer(
+            consumerData.getGroupName(),
+            clientInfo,
+            consumerData.getConsumeType(),
+            consumerData.getMessageModel(),
+            consumerData.getConsumeFromWhere(),
+            consumerData.getSubscriptionDataSet(),
+            false);
+        ackMessageProcessor = new AckMessageProcessor(brokerController);
+    }
+
+    @Test
+    public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException {
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+
+        int queueId = 0;
+        long queueOffset = 0;
+        long popTime = System.currentTimeMillis() - 1_000;
+        long invisibleTime = 30_000;
+        int reviveQid = 0;
+        String brokerName = "test_broker";
+        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,
+            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;
+        AckMessageRequestHeader requestHeader = new AckMessageRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(0);
+        requestHeader.setOffset(0L);
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setExtraInfo(extraInfo);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand responseToReturn = ackMessageProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java

@@ -20,15 +20,20 @@ import com.alibaba.fastjson.JSON;
 import com.google.common.collect.Sets;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.net.SocketAddress;
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
 import org.apache.rocketmq.broker.client.ConsumerManager;
 import org.apache.rocketmq.broker.offset.ConsumerOffsetManager;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
 import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.TopicQueueId;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
 import org.apache.rocketmq.common.message.MessageAccessor;
@@ -44,6 +49,7 @@ import org.apache.rocketmq.common.protocol.header.GetAllTopicConfigResponseHeade
 import org.apache.rocketmq.common.protocol.header.GetEarliestMsgStoretimeRequestHeader;
 import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader;
 import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader;
+import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader;
 import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader;
 import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
@@ -54,18 +60,12 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyClientConfig;
 import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.store.AppendMessageResult;
-import org.apache.rocketmq.store.AppendMessageStatus;
 import org.apache.rocketmq.store.DefaultMessageStore;
-import org.apache.rocketmq.store.MappedFile;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
-import org.apache.rocketmq.store.PutMessageResult;
-import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
-import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+import org.apache.rocketmq.store.logfile.DefaultMappedFile;
 import org.apache.rocketmq.store.stats.BrokerStats;
 import org.junit.Before;
 import org.junit.Test;
@@ -74,12 +74,12 @@ import org.mockito.Mock;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
-import java.net.SocketAddress;
+import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.LongAdder;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
@@ -97,18 +97,26 @@ public class AdminBrokerProcessorTest {
     @Mock
     private ChannelHandlerContext handlerContext;
 
+    @Mock
+    private Channel channel;
+
     @Spy
     private BrokerController
-            brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
-            new MessageStoreConfig());
+        brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(),
+        new MessageStoreConfig());
 
     @Mock
     private MessageStore messageStore;
 
-    private Set<String> systemTopicSet;
+    @Mock
+    private SendMessageProcessor sendMessageProcessor;
 
     @Mock
-    private Channel channel;
+    private ConcurrentMap<TopicQueueId, LongAdder> inFlyWritingCouterMap;
+
+    private Set<String> systemTopicSet;
+    private String topic;
+
     @Mock
     private SocketAddress socketAddress;
     @Mock
@@ -125,29 +133,35 @@ public class AdminBrokerProcessorTest {
     private ScheduleMessageService scheduleMessageService;
 
     @Before
-    public void init() {
+    public void init() throws Exception {
         brokerController.setMessageStore(messageStore);
+
+        //doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor();
+
         adminBrokerProcessor = new AdminBrokerProcessor(brokerController);
 
         systemTopicSet = Sets.newHashSet(
-                TopicValidator.RMQ_SYS_SELF_TEST_TOPIC,
-                TopicValidator.RMQ_SYS_BENCHMARK_TOPIC,
-                TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
-                TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT,
-                TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC,
-                this.brokerController.getBrokerConfig().getBrokerClusterName(),
-                this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX);
+            TopicValidator.RMQ_SYS_SELF_TEST_TOPIC,
+            TopicValidator.RMQ_SYS_BENCHMARK_TOPIC,
+            TopicValidator.RMQ_SYS_SCHEDULE_TOPIC,
+            TopicValidator.RMQ_SYS_OFFSET_MOVED_EVENT,
+            TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC,
+            this.brokerController.getBrokerConfig().getBrokerClusterName(),
+            this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX);
         if (this.brokerController.getBrokerConfig().isTraceTopicEnable()) {
             systemTopicSet.add(this.brokerController.getBrokerConfig().getMsgTraceTopicName());
         }
+        when(handlerContext.channel()).thenReturn(channel);
+        when(channel.remoteAddress()).thenReturn(new InetSocketAddress("127.0.0.1", 12345));
+
+        topic = "FooBar" + System.nanoTime();
+
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));
     }
 
     @Test
     public void testProcessRequest_success() throws RemotingCommandException, UnknownHostException {
-        RemotingCommand request = createResumeCheckHalfMessageCommand();
-        when(messageStore.selectOneMessageByOffset(any(Long.class))).thenReturn(createSelectMappedBufferResult());
-        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
-                (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+        RemotingCommand request = createUpdateBrokerConfigCommand();
         RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
     }
@@ -156,8 +170,6 @@ public class AdminBrokerProcessorTest {
     public void testProcessRequest_fail() throws RemotingCommandException, UnknownHostException {
         RemotingCommand request = createResumeCheckHalfMessageCommand();
         when(messageStore.selectOneMessageByOffset(any(Long.class))).thenReturn(createSelectMappedBufferResult());
-        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult
-                (PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)));
         RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
     }
@@ -494,14 +506,41 @@ public class AdminBrokerProcessorTest {
     public void testGetAllDelayOffset() throws Exception {
         defaultMessageStore = mock(DefaultMessageStore.class);
         scheduleMessageService = mock(ScheduleMessageService.class);
-        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
-        when(defaultMessageStore.getScheduleMessageService()).thenReturn(scheduleMessageService);
+//        when(brokerController.getMessageStore()).thenReturn(defaultMessageStore);
+        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);
         when(scheduleMessageService.encode()).thenReturn("content");
         RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null);
         RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
         assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
     }
 
+
+    @Test
+    public void testGetTopicConfig() throws Exception {
+        String topic = "foobar";
+
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig(topic));
+
+        {
+            GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader();
+            requestHeader.setTopic(topic);
+            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);
+            request.makeCustomHeaderToNet();
+            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+            assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+            assertThat(response.getBody()).isNotEmpty();
+        }
+        {
+            GetTopicConfigRequestHeader requestHeader = new GetTopicConfigRequestHeader();
+            requestHeader.setTopic("aaaaaaa");
+            RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);
+            request.makeCustomHeaderToNet();
+            RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
+            assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
+            assertThat(response.getRemark()).contains("No topic in this broker.");
+        }
+    }
+
     private RemotingCommand buildCreateTopicRequest(String topic) {
         CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();
         requestHeader.setTopic(topic);
@@ -537,7 +576,7 @@ public class AdminBrokerProcessorTest {
     }
 
     private SelectMappedBufferResult createSelectMappedBufferResult() {
-        SelectMappedBufferResult result = new SelectMappedBufferResult(0, ByteBuffer.allocate(1024), 0, new MappedFile());
+        SelectMappedBufferResult result = new SelectMappedBufferResult(0, ByteBuffer.allocate(1024), 0, new DefaultMappedFile());
         return result;
     }
 
@@ -553,4 +592,10 @@ public class AdminBrokerProcessorTest {
         request.makeCustomHeaderToNet();
         return request;
     }
+
+    private RemotingCommand createUpdateBrokerConfigCommand() {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
 }

broker/src/test/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessorTest.java

@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.lang.reflect.Field;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.client.net.Broker2Client;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader;
+import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ChangeInvisibleTimeProcessorTest {
+    private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessor;
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private DefaultMessageStore messageStore;
+    @Mock
+    private Channel channel;
+
+    private String topic = "FooBar";
+    private String group = "FooBarGroup";
+    private ClientChannelInfo clientInfo;
+    @Mock
+    private Broker2Client broker2Client;
+
+    @Before
+    public void init() throws IllegalAccessException, NoSuchFieldException {
+        brokerController.setMessageStore(messageStore);
+        Field field = BrokerController.class.getDeclaredField("broker2Client");
+        field.setAccessible(true);
+        field.set(brokerController, broker2Client);
+        ScheduleMessageService scheduleMessageService = new ScheduleMessageService(brokerController);
+        scheduleMessageService.parseDelayLevel();
+        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);
+        Channel mockChannel = mock(Channel.class);
+        when(handlerContext.channel()).thenReturn(mockChannel);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
+        ConsumerData consumerData = createConsumerData(group, topic);
+        clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0);
+        brokerController.getConsumerManager().registerConsumer(
+            consumerData.getGroupName(),
+            clientInfo,
+            consumerData.getConsumeType(),
+            consumerData.getMessageModel(),
+            consumerData.getConsumeFromWhere(),
+            consumerData.getSubscriptionDataSet(),
+            false);
+        changeInvisibleTimeProcessor = new ChangeInvisibleTimeProcessor(brokerController);
+    }
+
+    @Test
+    public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException {
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+        int queueId = 0;
+        long queueOffset = 0;
+        long popTime = System.currentTimeMillis() - 1_000;
+        long invisibleTime = 30_000;
+        int reviveQid = 0;
+        String brokerName = "test_broker";
+        String extraInfo = ExtraInfoUtil.buildExtraInfo(queueOffset, popTime, invisibleTime, reviveQid,
+            topic, brokerName, queueId) + MessageConst.KEY_SEPARATOR + queueOffset;
+
+        ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();
+        requestHeader.setTopic(topic);
+        requestHeader.setQueueId(queueId);
+        requestHeader.setOffset(queueOffset);
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setExtraInfo(extraInfo);
+        requestHeader.setInvisibleTime(invisibleTime);
+
+        final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader);
+        request.makeCustomHeaderToNet();
+        RemotingCommand responseToReturn = changeInvisibleTimeProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/ClientManageProcessorTest.java

@@ -18,7 +18,6 @@ package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.rocketmq.broker.BrokerController;

broker/src/test/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessorTest.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
+import org.apache.rocketmq.common.topic.TopicValidator;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumerManageProcessorTest {
+    private ConsumerManageProcessor consumerManageProcessor;
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    @Mock
+    private MessageStore messageStore;
+
+    private String topic = "FooBar";
+    private String group = "FooBarGroup";
+
+    @Before
+    public void init() {
+        brokerController.setMessageStore(messageStore);
+        TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController);
+        topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        consumerManageProcessor = new ConsumerManageProcessor(brokerController);
+    }
+
+    @Test
+    public void testUpdateConsumerOffset_InvalidTopic() throws Exception {
+        RemotingCommand request = createConsumerManageCommand(RequestCode.UPDATE_CONSUMER_OFFSET);
+        request.addExtField("topic", "InvalidTopic");
+        RemotingCommand response = consumerManageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
+    }
+
+    private RemotingCommand createConsumerManageCommand(int requestCode) {
+        SendMessageRequestHeader requestHeader = new SendMessageRequestHeader();
+        requestHeader.setProducerGroup(group);
+        requestHeader.setTopic(topic);
+        requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC);
+        requestHeader.setDefaultTopicQueueNums(3);
+        requestHeader.setQueueId(1);
+        requestHeader.setSysFlag(0);
+        requestHeader.setBornTimestamp(System.currentTimeMillis());
+        requestHeader.setFlag(124);
+        requestHeader.setReconsumeTimes(0);
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader);
+        request.setBody(new byte[] {'a'});
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/EndTransactionProcessorTest.java

@@ -34,7 +34,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.AppendMessageResult;
 import org.apache.rocketmq.store.AppendMessageStatus;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;

broker/src/test/java/org/apache/rocketmq/broker/processor/PopBufferMergeServiceTest.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.pop.AckMsg;
+import org.apache.rocketmq.store.pop.PopCheckPoint;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PopBufferMergeServiceTest {
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    @Mock
+    private PopMessageProcessor popMessageProcessor;
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private DefaultMessageStore messageStore;
+    private ScheduleMessageService scheduleMessageService;
+    private ClientChannelInfo clientChannelInfo;
+    private String group = "FooBarGroup";
+    private String topic = "FooBar";
+
+    @Before
+    public void init() throws Exception {
+        FieldUtils.writeField(brokerController.getBrokerConfig(), "enablePopBufferMerge", true, true);
+        brokerController.setMessageStore(messageStore);
+        popMessageProcessor = new PopMessageProcessor(brokerController);
+        scheduleMessageService = new ScheduleMessageService(brokerController);
+        scheduleMessageService.parseDelayLevel();
+        Channel mockChannel = mock(Channel.class);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
+        clientChannelInfo = new ClientChannelInfo(mockChannel);
+        ConsumerData consumerData = createConsumerData(group, topic);
+        brokerController.getConsumerManager().registerConsumer(
+            consumerData.getGroupName(),
+            clientChannelInfo,
+            consumerData.getConsumeType(),
+            consumerData.getMessageModel(),
+            consumerData.getConsumeFromWhere(),
+            consumerData.getSubscriptionDataSet(),
+            false);
+    }
+
+    @Test(timeout = 10_000)
+    public void testBasic() throws Exception {
+        PopBufferMergeService popBufferMergeService = new PopBufferMergeService(brokerController, popMessageProcessor);
+        popBufferMergeService.start();
+        PopCheckPoint ck = new PopCheckPoint();
+        ck.setBitMap(0);
+        int msgCnt = 1;
+        ck.setNum((byte) msgCnt);
+        long popTime = System.currentTimeMillis() - 1000;
+        ck.setPopTime(popTime);
+        int invisibleTime = 30_000;
+        ck.setInvisibleTime(invisibleTime);
+        int offset = 100;
+        ck.setStartOffset(offset);
+        ck.setCId(group);
+        ck.setTopic(topic);
+        int queueId = 0;
+        ck.setQueueId((byte) queueId);
+
+        int reviveQid = 0;
+        long nextBeginOffset = 101L;
+        long ackOffset = offset;
+        AckMsg ackMsg = new AckMsg();
+        ackMsg.setAckOffset(ackOffset);
+        ackMsg.setStartOffset(offset);
+        ackMsg.setConsumerGroup(group);
+        ackMsg.setTopic(topic);
+        ackMsg.setQueueId(queueId);
+        ackMsg.setPopTime(popTime);
+        try {
+            assertThat(popBufferMergeService.addCk(ck, reviveQid, ackOffset, nextBeginOffset)).isTrue();
+            assertThat(popBufferMergeService.getLatestOffset(topic, group, queueId)).isEqualTo(nextBeginOffset);
+            Thread.sleep(1000); // wait background threads of PopBufferMergeService run for some time
+            assertThat(popBufferMergeService.addAk(reviveQid, ackMsg)).isTrue();
+            assertThat(popBufferMergeService.getLatestOffset(topic, group, queueId)).isEqualTo(nextBeginOffset);
+        } finally {
+            popBufferMergeService.shutdown(true);
+        }
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java

@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.schedule.ScheduleMessageService;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.constant.ConsumeInitMode;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.logfile.DefaultMappedFile;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class PopMessageProcessorTest {
+    private PopMessageProcessor popMessageProcessor;
+
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private DefaultMessageStore messageStore;
+    private ScheduleMessageService scheduleMessageService;
+    private ClientChannelInfo clientChannelInfo;
+    private String group = "FooBarGroup";
+    private String topic = "FooBar";
+
+    @Before
+    public void init() {
+        brokerController.setMessageStore(messageStore);
+        popMessageProcessor = new PopMessageProcessor(brokerController);
+        scheduleMessageService = new ScheduleMessageService(brokerController);
+        scheduleMessageService.parseDelayLevel();
+        when(brokerController.getScheduleMessageService()).thenReturn(scheduleMessageService);
+        when(messageStore.putMessage(any())).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+        Channel mockChannel = mock(Channel.class);
+        when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024));
+        when(handlerContext.channel()).thenReturn(mockChannel);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
+        clientChannelInfo = new ClientChannelInfo(mockChannel);
+        ConsumerData consumerData = createConsumerData(group, topic);
+        brokerController.getConsumerManager().registerConsumer(
+            consumerData.getGroupName(),
+            clientChannelInfo,
+            consumerData.getConsumeType(),
+            consumerData.getMessageModel(),
+            consumerData.getConsumeFromWhere(),
+            consumerData.getSubscriptionDataSet(),
+            false);
+    }
+
+    @Test
+    public void testProcessRequest_TopicNotExist() throws RemotingCommandException {
+        brokerController.getTopicConfigManager().getTopicConfigTable().remove(topic);
+        final RemotingCommand request = createPopMsgCommand();
+        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
+        assertThat(response.getRemark()).contains("topic[" + topic + "] not exist");
+    }
+
+    @Test
+    public void testProcessRequest_SubNotExist() throws RemotingCommandException {
+        brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, false);
+        final RemotingCommand request = createPopMsgCommand();
+        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_EXIST);
+        assertThat(response.getRemark()).contains("consumer's group info not exist");
+    }
+
+    @Test
+    public void testProcessRequest_Found() throws RemotingCommandException {
+        GetMessageResult getMessageResult = createGetMessageResult(1);
+        when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult);
+
+        final RemotingCommand request = createPopMsgCommand();
+        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testProcessRequest_MsgWasRemoving() throws RemotingCommandException {
+        GetMessageResult getMessageResult = createGetMessageResult(1);
+        getMessageResult.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING);
+        when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult);
+
+        final RemotingCommand request = createPopMsgCommand();
+        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testProcessRequest_NoMsgInQueue() throws RemotingCommandException {
+        GetMessageResult getMessageResult = createGetMessageResult(0);
+        getMessageResult.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE);
+        when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(getMessageResult);
+
+        final RemotingCommand request = createPopMsgCommand();
+        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNull();
+    }
+
+
+    private RemotingCommand createPopMsgCommand() {
+        PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();
+        requestHeader.setConsumerGroup(group);
+        requestHeader.setMaxMsgNums(30);
+        requestHeader.setQueueId(-1);
+        requestHeader.setTopic(topic);
+        requestHeader.setInvisibleTime(10_000);
+        requestHeader.setInitMode(ConsumeInitMode.MAX);
+        requestHeader.setOrder(false);
+        requestHeader.setPollTime(15_000);
+        requestHeader.setBornTime(System.currentTimeMillis());
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader);
+        request.makeCustomHeaderToNet();
+        return request;
+    }
+
+
+    private GetMessageResult createGetMessageResult(int msgCnt) {
+        GetMessageResult getMessageResult = new GetMessageResult();
+        getMessageResult.setStatus(GetMessageStatus.FOUND);
+        getMessageResult.setMinOffset(100);
+        getMessageResult.setMaxOffset(1024);
+        getMessageResult.setNextBeginOffset(516);
+        for (int i = 0; i < msgCnt; i++) {
+            ByteBuffer bb = ByteBuffer.allocate(64);
+            bb.putLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSITION, System.currentTimeMillis());
+            getMessageResult.addMessage(new SelectMappedBufferResult(200, bb, 64, new DefaultMappedFile()));
+        }
+        return getMessageResult;
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java

@@ -18,16 +18,12 @@ package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
 import org.apache.rocketmq.broker.BrokerController;
 import org.apache.rocketmq.broker.client.ClientChannelInfo;
 import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
 import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
 import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
@@ -53,6 +49,12 @@ import org.mockito.Mock;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyInt;
@@ -65,7 +67,8 @@ import static org.mockito.Mockito.when;
 public class PullMessageProcessorTest {
     private PullMessageProcessor pullMessageProcessor;
     @Spy
-    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
+        new NettyClientConfig(), new MessageStoreConfig());
     @Mock
     private ChannelHandlerContext handlerContext;
     @Mock
@@ -77,10 +80,14 @@ public class PullMessageProcessorTest {
     @Before
     public void init() {
         brokerController.setMessageStore(messageStore);
+        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);
         pullMessageProcessor = new PullMessageProcessor(brokerController);
         Channel mockChannel = mock(Channel.class);
+        when(mockChannel.isWritable()).thenReturn(true);
         when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024));
         when(handlerContext.channel()).thenReturn(mockChannel);
+        when(handlerContext.channel().isWritable()).thenReturn(true);
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
         brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
         clientChannelInfo = new ClientChannelInfo(mockChannel);
         ConsumerData consumerData = createConsumerData(group, topic);
@@ -192,6 +199,15 @@ public class PullMessageProcessorTest {
         assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_OFFSET_MOVED);
     }
 
+    @Test
+    public void test_LitePullRequestForbidden() throws Exception {
+        brokerController.getBrokerConfig().setLitePullMessageEnable(false);
+        RemotingCommand remotingCommand = createPullMsgCommand(RequestCode.LITE_PULL_MESSAGE);
+        RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, remotingCommand);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+    }
+
     private RemotingCommand createPullMsgCommand(int requestCode) {
         PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
         requestHeader.setCommitOffset(123L);

broker/src/test/java/org/apache/rocketmq/broker/processor/QueryAssignmentProcessorTest.java

@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.processor;
+
+import com.google.common.collect.ImmutableSet;
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelHandlerContext;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.client.ClientChannelInfo;
+import org.apache.rocketmq.broker.loadbalance.AssignmentManager;
+import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle;
+import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueConsistentHash;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.message.MessageRequestMode;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody;
+import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
+import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.netty.NettyClientConfig;
+import org.apache.rocketmq.remoting.netty.NettyServerConfig;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.apache.rocketmq.broker.processor.PullMessageProcessorTest.createConsumerData;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class QueryAssignmentProcessorTest {
+    private QueryAssignmentProcessor queryAssignmentProcessor;
+    @Spy
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+
+    @Mock
+    private AssignmentManager assignmentManager;
+    @Mock
+    private ChannelHandlerContext handlerContext;
+    @Mock
+    private MessageStore messageStore;
+    @Mock
+    private Channel channel;
+
+    private String broker = "defaultBroker";
+    private String topic = "FooBar";
+    private String group = "FooBarGroup";
+    private String clientId = "127.0.0.1";
+    private ClientChannelInfo clientInfo;
+
+    @Before
+    public void init() throws IllegalAccessException, NoSuchFieldException {
+        clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0);
+        brokerController.setMessageStore(messageStore);
+        doReturn(assignmentManager).when(brokerController).getAssignmentManager();
+        when(assignmentManager.getTopicSubscribeInfo(topic)).thenReturn(ImmutableSet.of(new MessageQueue(topic, "broker-1", 0), new MessageQueue(topic, "broker-2", 1)));
+        queryAssignmentProcessor = new QueryAssignmentProcessor(brokerController);
+        brokerController.getTopicConfigManager().getTopicConfigTable().put(topic, new TopicConfig());
+        ConsumerData consumerData = createConsumerData(group, topic);
+        brokerController.getConsumerManager().registerConsumer(
+            consumerData.getGroupName(),
+            clientInfo,
+            consumerData.getConsumeType(),
+            consumerData.getMessageModel(),
+            consumerData.getConsumeFromWhere(),
+            consumerData.getSubscriptionDataSet(),
+            false);
+    }
+
+    @Test
+    public void testQueryAssignment() throws Exception {
+        brokerController.getProducerManager().registerProducer(group, clientInfo);
+        final RemotingCommand request = createQueryAssignmentRequest();
+        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
+        assertThat(responseToReturn.getBody()).isNotNull();
+        QueryAssignmentResponseBody responseBody = QueryAssignmentResponseBody.decode(responseToReturn.getBody(), QueryAssignmentResponseBody.class);
+        assertThat(responseBody.getMessageQueueAssignments()).size().isEqualTo(2);
+    }
+
+    @Test
+    public void testSetMessageRequestMode_Success() throws Exception {
+        brokerController.getProducerManager().registerProducer(group, clientInfo);
+        final RemotingCommand request = createSetMessageRequestModeRequest(topic);
+        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
+    @Test
+    public void testSetMessageRequestMode_RetryTopic() throws Exception {
+        brokerController.getProducerManager().registerProducer(group, clientInfo);
+        final RemotingCommand request = createSetMessageRequestModeRequest(MixAll.RETRY_GROUP_TOPIC_PREFIX + topic);
+        RemotingCommand responseToReturn = queryAssignmentProcessor.processRequest(handlerContext, request);
+        assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
+    }
+
+    @Test
+    public void testAllocate4Pop() {
+        testAllocate4Pop(new AllocateMessageQueueAveragely());
+        testAllocate4Pop(new AllocateMessageQueueAveragelyByCircle());
+        testAllocate4Pop(new AllocateMessageQueueConsistentHash());
+    }
+
+    private void testAllocate4Pop(AllocateMessageQueueStrategy strategy) {
+        int testNum = 16;
+        List<MessageQueue> mqAll = new ArrayList<>();
+        for (int mqSize = 0; mqSize < testNum; mqSize++) {
+            mqAll.add(new MessageQueue(topic, broker, mqSize));
+
+            List<String> cidAll = new ArrayList<>();
+            for (int cidSize = 0; cidSize < testNum; cidSize++) {
+                String clientId = String.valueOf(cidSize);
+                cidAll.add(clientId);
+
+                for (int popShareQueueNum = 0; popShareQueueNum < testNum; popShareQueueNum++) {
+                    List<MessageQueue> allocateResult =
+                        queryAssignmentProcessor.allocate4Pop(strategy, group, clientId, mqAll, cidAll, popShareQueueNum);
+                    Assert.assertTrue(checkAllocateResult(popShareQueueNum, mqAll.size(), cidAll.size(), allocateResult.size(), strategy));
+                }
+            }
+        }
+    }
+
+    private boolean checkAllocateResult(int popShareQueueNum, int mqSize, int cidSize, int allocateSize,
+        AllocateMessageQueueStrategy strategy) {
+
+        //The maximum size of allocations will not exceed mqSize.
+        if (allocateSize > mqSize) {
+            return false;
+        }
+
+        //It is not allowed that the client is not assigned to the consumeQueue.
+        if (allocateSize <= 0) {
+            return false;
+        }
+
+        if (popShareQueueNum <= 0 || popShareQueueNum >= cidSize - 1) {
+            return allocateSize == mqSize;
+        } else if (mqSize < cidSize) {
+            return allocateSize == 1;
+        }
+
+        if (strategy instanceof AllocateMessageQueueAveragely
+            || strategy instanceof AllocateMessageQueueAveragelyByCircle) {
+
+            if (mqSize % cidSize == 0) {
+                return allocateSize == (mqSize / cidSize) * (popShareQueueNum + 1);
+            } else {
+                int avgSize = mqSize / cidSize;
+                return allocateSize >= avgSize * (popShareQueueNum + 1)
+                    && allocateSize <= (avgSize + 1) * (popShareQueueNum + 1);
+            }
+        }
+
+        if (strategy instanceof AllocateMessageQueueConsistentHash) {
+            //Just skip
+            return true;
+        }
+
+        return false;
+    }
+
+    private RemotingCommand createQueryAssignmentRequest() {
+        QueryAssignmentRequestBody requestBody = new QueryAssignmentRequestBody();
+        requestBody.setTopic(topic);
+        requestBody.setConsumerGroup(group);
+        requestBody.setClientId(clientId);
+        requestBody.setMessageModel(MessageModel.CLUSTERING);
+        requestBody.setStrategyName("AVG");
+
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_ASSIGNMENT, null);
+        request.setBody(requestBody.encode());
+        return request;
+    }
+
+    private RemotingCommand createSetMessageRequestModeRequest(String topic) {
+        RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SET_MESSAGE_REQUEST_MODE, null);
+
+        SetMessageRequestModeRequestBody requestBody = new SetMessageRequestModeRequestBody();
+        requestBody.setTopic(topic);
+        requestBody.setConsumerGroup(group);
+        requestBody.setMode(MessageRequestMode.POP);
+        requestBody.setPopShareQueueNum(0);
+        request.setBody(requestBody.encode());
+
+        return request;
+    }
+
+    private RemotingCommand createResponse(int code, RemotingCommand request) {
+        RemotingCommand response = RemotingCommand.createResponseCommand(null);
+        response.setCode(code);
+        response.setOpaque(request.getOpaque());
+        return response;
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java

@@ -42,7 +42,7 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.AppendMessageResult;
 import org.apache.rocketmq.store.AppendMessageStatus;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;

broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java

@@ -18,11 +18,19 @@ package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandlerContext;
+import java.time.Duration;
+import java.util.concurrent.Executors;
 import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.mqtrace.AbortProcessException;
+import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
+import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
 import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
 import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
+import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
+import org.apache.rocketmq.broker.topic.TopicConfigManager;
 import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
 import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
@@ -38,7 +46,7 @@ import org.apache.rocketmq.remoting.netty.NettyServerConfig;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 import org.apache.rocketmq.store.AppendMessageResult;
 import org.apache.rocketmq.store.AppendMessageStatus;
-import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
@@ -59,6 +67,7 @@ import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.Mockito.doAnswer;
@@ -71,7 +80,8 @@ public class SendMessageProcessorTest {
     @Mock
     private ChannelHandlerContext handlerContext;
     @Spy
-    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig());
+    private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(),
+        new NettyClientConfig(), new MessageStoreConfig());
     @Mock
     private MessageStore messageStore;
 
@@ -84,27 +94,31 @@ public class SendMessageProcessorTest {
     @Before
     public void init() {
         brokerController.setMessageStore(messageStore);
+        TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController);
+        topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));
+        SubscriptionGroupManager subscriptionGroupManager = new SubscriptionGroupManager(brokerController);
+        when(brokerController.getSubscriptionGroupManager()).thenReturn(subscriptionGroupManager);
+        when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager);
+        when(brokerController.getPutMessageFutureExecutor()).thenReturn(Executors.newSingleThreadExecutor());
         when(messageStore.now()).thenReturn(System.currentTimeMillis());
         Channel mockChannel = mock(Channel.class);
         when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024));
         when(handlerContext.channel()).thenReturn(mockChannel);
-        MessageExt messageExt = new MessageExt();
-        messageExt.setTopic(topic);
-        when(messageStore.lookMessageByOffset(anyLong())).thenReturn(messageExt);
+        when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt());
         sendMessageProcessor = new SendMessageProcessor(brokerController);
     }
 
     @Test
-    public void testProcessRequest() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
+    public void testProcessRequest() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
         assertPutResult(ResponseCode.SUCCESS);
     }
 
     @Test
-    public void testProcessRequest_WithHook() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
+    public void testProcessRequest_WithHook() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
         List<SendMessageHook> sendMessageHookList = new ArrayList<>();
         final SendMessageContext[] sendMessageContext = new SendMessageContext[1];
         SendMessageHook sendMessageHook = new SendMessageHook() {
@@ -133,65 +147,65 @@ public class SendMessageProcessorTest {
     }
 
     @Test
-    public void testProcessRequest_FlushTimeOut() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_FlushTimeOut() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_DISK_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.FLUSH_DISK_TIMEOUT);
     }
 
     @Test
-    public void testProcessRequest_MessageIllegal() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_MessageIllegal() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.MESSAGE_ILLEGAL);
     }
 
     @Test
-    public void testProcessRequest_CreateMappedFileFailed() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_CreateMappedFileFailed() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.CREATE_MAPPED_FILE_FAILED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.SYSTEM_ERROR);
     }
 
     @Test
-    public void testProcessRequest_FlushSlaveTimeout() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_FlushSlaveTimeout() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.FLUSH_SLAVE_TIMEOUT);
     }
 
     @Test
-    public void testProcessRequest_PageCacheBusy() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_PageCacheBusy() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.OS_PAGE_CACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.SYSTEM_ERROR);
     }
 
     @Test
-    public void testProcessRequest_PropertiesTooLong() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_PropertiesTooLong() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PROPERTIES_SIZE_EXCEEDED, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.MESSAGE_ILLEGAL);
     }
 
     @Test
-    public void testProcessRequest_ServiceNotAvailable() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_ServiceNotAvailable() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.SERVICE_NOT_AVAILABLE);
     }
 
     @Test
-    public void testProcessRequest_SlaveNotAvailable() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SLAVE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
+    public void testProcessRequest_SlaveNotAvailable() throws Exception {
+        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.SLAVE_NOT_AVAILABLE, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))));
         assertPutResult(ResponseCode.SLAVE_NOT_AVAILABLE);
     }
 
     @Test
-    public void testProcessRequest_WithMsgBack() throws RemotingCommandException {
-        when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
+    public void testProcessRequest_WithMsgBack() throws Exception {
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
         final RemotingCommand request = createSendMsgBackCommand(RequestCode.CONSUMER_SEND_MSG_BACK);
 
         sendMessageProcessor = new SendMessageProcessor(brokerController);
@@ -204,18 +218,95 @@ public class SendMessageProcessorTest {
     public void testProcessRequest_Transaction() throws RemotingCommandException {
         brokerController.setTransactionalMessageService(transactionMsgService);
         when(brokerController.getTransactionalMessageService().asyncPrepareMessage(any(MessageExtBrokerInner.class)))
-                .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
+            .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
         RemotingCommand request = createSendTransactionMsgCommand(RequestCode.SEND_MESSAGE);
         final RemotingCommand[] response = new RemotingCommand[1];
+        doAnswer(new Answer() {
+            @Override
+            public Object answer(InvocationOnMock invocation) throws Throwable {
+                response[0] = invocation.getArgument(0);
+                return null;
+            }
+        }).when(handlerContext).writeAndFlush(any(Object.class));
+        await().atMost(Duration.ofSeconds(10)).until(() -> {
+            RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
+            if (responseToReturn != null) {
+                assertThat(response[0]).isNull();
+                response[0] = responseToReturn;
+            }
 
-        RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
-        if (responseToReturn != null) {
-            assertThat(response[0]).isNull();
-            response[0] = responseToReturn;
-        }
-        assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS);
-
+            if (response[0] == null) {
+                return false;
+            }
+            assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS);
+            assertThat(response[0].getOpaque()).isEqualTo(request.getOpaque());
+            return true;
+        });
     }
+
+    @Test
+    public void testProcessRequest_WithAbortProcessSendMessageBeforeHook() throws Exception {
+        List<SendMessageHook> sendMessageHookList = new ArrayList<>();
+        final SendMessageContext[] sendMessageContext = new SendMessageContext[1];
+        SendMessageHook sendMessageHook = new SendMessageHook() {
+            @Override
+            public String hookName() {
+                return null;
+            }
+
+            @Override
+            public void sendMessageBefore(SendMessageContext context) {
+                sendMessageContext[0] = context;
+                throw new AbortProcessException(ResponseCode.FLOW_CONTROL, "flow control test");
+            }
+
+            @Override
+            public void sendMessageAfter(SendMessageContext context) {
+
+            }
+        };
+        sendMessageHookList.add(sendMessageHook);
+        sendMessageProcessor.registerSendMessageHook(sendMessageHookList);
+        assertPutResult(ResponseCode.FLOW_CONTROL);
+        System.out.println(sendMessageContext[0]);
+        assertThat(sendMessageContext[0]).isNotNull();
+        assertThat(sendMessageContext[0].getTopic()).isEqualTo(topic);
+        assertThat(sendMessageContext[0].getProducerGroup()).isEqualTo(group);
+    }
+
+    @Test
+    public void testProcessRequest_WithMsgBackWithConsumeMessageAfterHook() throws Exception {
+        when(messageStore.putMessage(any(MessageExtBrokerInner.class))).
+            thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)));
+        final RemotingCommand request = createSendMsgBackCommand(RequestCode.CONSUMER_SEND_MSG_BACK);
+
+        sendMessageProcessor = new SendMessageProcessor(brokerController);
+        List<ConsumeMessageHook> consumeMessageHookList = new ArrayList<>();
+        final ConsumeMessageContext[] messageContext = new ConsumeMessageContext[1];
+        ConsumeMessageHook consumeMessageHook = new ConsumeMessageHook() {
+            @Override
+            public String hookName() {
+                return "TestHook";
+            }
+
+            @Override
+            public void consumeMessageBefore(ConsumeMessageContext context) {
+
+            }
+
+            @Override
+            public void consumeMessageAfter(ConsumeMessageContext context) {
+                messageContext[0] = context;
+                throw new AbortProcessException(ResponseCode.FLOW_CONTROL, "flow control test");
+            }
+        };
+        consumeMessageHookList.add(consumeMessageHook);
+        sendMessageProcessor.registerConsumeMessageHook(consumeMessageHookList);
+        final RemotingCommand response = sendMessageProcessor.processRequest(handlerContext, request);
+        assertThat(response).isNotNull();
+        assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
+    }
+
     private RemotingCommand createSendTransactionMsgCommand(int requestCode) {
         SendMessageRequestHeader header = createSendMsgRequestHeader();
         int sysFlag = header.getSysFlag();
@@ -266,15 +357,37 @@ public class SendMessageProcessorTest {
         return request;
     }
 
+    /**
+     * We will explain the logic of this method so you can get a better feeling of how to use it: This method assumes
+     * that if responseToReturn is not null, then there would be an error, which means the writeAndFlush are never
+     * reached. If responseToReturn is null, means everything ok, so writeAndFlush should record the actual response.
+     *
+     * @param responseCode
+     * @throws RemotingCommandException
+     */
     private void assertPutResult(int responseCode) throws RemotingCommandException {
         final RemotingCommand request = createSendMsgCommand(RequestCode.SEND_MESSAGE);
         final RemotingCommand[] response = new RemotingCommand[1];
-        RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
-        if (responseToReturn != null) {
-            assertThat(response[0]).isNull();
-            response[0] = responseToReturn;
-        }
-        assertThat(response[0].getCode()).isEqualTo(responseCode);
-        assertThat(response[0].getOpaque()).isEqualTo(request.getOpaque());
+        doAnswer(new Answer() {
+            @Override
+            public Object answer(InvocationOnMock invocation) throws Throwable {
+                response[0] = invocation.getArgument(0);
+                return null;
+            }
+        }).when(handlerContext).writeAndFlush(any(Object.class));
+        await().atMost(Duration.ofSeconds(10)).until(() -> {
+            RemotingCommand responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
+            if (responseToReturn != null) {
+                assertThat(response[0]).isNull();
+                response[0] = responseToReturn;
+            }
+
+            if (response[0] == null) {
+                return false;
+            }
+            assertThat(response[0].getCode()).isEqualTo(responseCode);
+            assertThat(response[0].getOpaque()).isEqualTo(request.getOpaque());
+            return true;
+        });
     }
 }