Prepare to release 4.5.0

.gitignore

@@ -10,6 +10,6 @@ devenv
 *.versionsBackup
 !NOTICE-BIN
 !LICENSE-BIN
-.DS_Store
-localbin
-nohup.out
+.DS_Store
+localbin
+nohup.out

README.md

@@ -49,3 +49,5 @@ We always welcome new contributions, whether for trivial cleanups, [big new feat
 ----------
 ## License
 [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation
+
+

broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java

@@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -39,6 +40,7 @@ import org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener;
 import org.apache.rocketmq.broker.client.ProducerManager;
 import org.apache.rocketmq.broker.client.net.Broker2Client;
 import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
+import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;
 import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.broker.filtersrv.FilterServerManager;
@@ -99,6 +101,7 @@ import org.apache.rocketmq.store.MessageArrivingListener;
 import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
 import org.apache.rocketmq.store.stats.BrokerStats;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
@@ -158,6 +161,7 @@ public class BrokerController {
     private TransactionalMessageCheckService transactionalMessageCheckService;
     private TransactionalMessageService transactionalMessageService;
     private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;
+    private Future<?> slaveSyncFuture;
 
 
     public BrokerController(
@@ -234,6 +238,10 @@ public class BrokerController {
                 this.messageStore =
                     new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener,
                         this.brokerConfig);
+                if (messageStoreConfig.isEnableDLegerCommitLog()) {
+                    DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore);
+                    ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);
+                }
                 this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
                 //load plugin
                 MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig);
@@ -396,37 +404,26 @@ public class BrokerController {
                 }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS);
             }
 
-            if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
-                if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
-                    this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
-                    this.updateMasterHAServerAddrPeriodically = false;
+            if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+                if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
+                    if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) {
+                        this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress());
+                        this.updateMasterHAServerAddrPeriodically = false;
+                    } else {
+                        this.updateMasterHAServerAddrPeriodically = true;
+                    }
                 } else {
-                    this.updateMasterHAServerAddrPeriodically = true;
+                    this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+                        @Override
+                        public void run() {
+                            try {
+                                BrokerController.this.printMasterAndSlaveDiff();
+                            } catch (Throwable e) {
+                                log.error("schedule printMasterAndSlaveDiff error.", e);
+                            }
+                        }
+                    }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
                 }
-
-                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        try {
-                            BrokerController.this.slaveSynchronize.syncAll();
-                        } catch (Throwable e) {
-                            log.error("ScheduledTask syncAll slave exception", e);
-                        }
-                    }
-                }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
-            } else {
-                this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
-
-                    @Override
-                    public void run() {
-                        try {
-                            BrokerController.this.printMasterAndSlaveDiff();
-                        } catch (Throwable e) {
-                            log.error("schedule printMasterAndSlaveDiff error.", e);
-                        }
-                    }
-                }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS);
             }
 
             if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {
@@ -855,6 +852,13 @@ public class BrokerController {
             this.filterServerManager.start();
         }
 
+        if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+            startProcessorByHa(messageStoreConfig.getBrokerRole());
+            handleSlaveSynchronize(messageStoreConfig.getBrokerRole());
+        }
+
+
+
         this.registerBrokerAll(true, false, true);
 
         this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
@@ -877,12 +881,7 @@ public class BrokerController {
             this.brokerFastFailure.start();
         }
 
-        if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
-            if (this.transactionalMessageCheckService != null) {
-                log.info("Start transaction service!");
-                this.transactionalMessageCheckService.start();
-            }
-        }
+
     }
 
     public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) {
@@ -1101,4 +1100,118 @@ public class BrokerController {
         return endTransactionThreadPoolQueue;
 
     }
+
+
+
+    private void handleSlaveSynchronize(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.slaveSynchronize.setMasterAddr(null);
+            slaveSyncFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
+                @Override
+                public void run() {
+                    try {
+                        BrokerController.this.slaveSynchronize.syncAll();
+                    }
+                    catch (Throwable e) {
+                        log.error("ScheduledTask SlaveSynchronize syncAll error.", e);
+                    }
+                }
+            }, 1000 * 3, 1000 * 10, TimeUnit.MILLISECONDS);
+        } else {
+            //handle the slave synchronise
+            if (null != slaveSyncFuture) {
+                slaveSyncFuture.cancel(false);
+            }
+            this.slaveSynchronize.setMasterAddr(null);
+        }
+    }
+
+    public void changeToSlave(int brokerId) {
+        log.info("Begin to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId);
+
+        //change the role
+        brokerConfig.setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check
+        messageStoreConfig.setBrokerRole(BrokerRole.SLAVE);
+
+        //handle the scheduled service
+        try {
+            this.messageStore.handleScheduleMessageService(BrokerRole.SLAVE);
+        } catch (Throwable t) {
+            log.error("[MONITOR] handleScheduleMessageService failed when changing to slave", t);
+        }
+
+        //handle the transactional service
+        try {
+            this.shutdownProcessorByHa();
+        } catch (Throwable t) {
+            log.error("[MONITOR] shutdownProcessorByHa failed when changing to slave", t);
+        }
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(BrokerRole.SLAVE);
+
+        try {
+            this.registerBrokerAll(true, true, brokerConfig.isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId);
+    }
+
+
+
+    public void changeToMaster(BrokerRole role) {
+        if (role == BrokerRole.SLAVE) {
+            return;
+        }
+        log.info("Begin to change to master brokerName={}", brokerConfig.getBrokerName());
+
+        //handle the slave synchronise
+        handleSlaveSynchronize(role);
+
+        //handle the scheduled service
+        try {
+            this.messageStore.handleScheduleMessageService(role);
+        } catch (Throwable t) {
+            log.error("[MONITOR] handleScheduleMessageService failed when changing to master", t);
+        }
+
+        //handle the transactional service
+        try {
+            this.startProcessorByHa(BrokerRole.SYNC_MASTER);
+        } catch (Throwable t) {
+            log.error("[MONITOR] startProcessorByHa failed when changing to master", t);
+        }
+
+        //if the operations above are totally successful, we change to master
+        brokerConfig.setBrokerId(0); //TO DO check
+        messageStoreConfig.setBrokerRole(role);
+
+        try {
+            this.registerBrokerAll(true, true, brokerConfig.isForceRegister());
+        } catch (Throwable ignored) {
+
+        }
+        log.info("Finish to change to master brokerName={}", brokerConfig.getBrokerName());
+    }
+
+    private void startProcessorByHa(BrokerRole role) {
+        if (BrokerRole.SLAVE != role) {
+            if (this.transactionalMessageCheckService != null) {
+                this.transactionalMessageCheckService.start();
+            }
+        }
+    }
+
+    private void shutdownProcessorByHa() {
+        if (this.transactionalMessageCheckService != null) {
+            this.transactionalMessageCheckService.shutdown(true);
+        }
+    }
+
+
+
 }

broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.dledger;
+
+import io.openmessaging.storage.dledger.DLedgerLeaderElector;
+import io.openmessaging.storage.dledger.DLedgerServer;
+import io.openmessaging.storage.dledger.MemberState;
+import io.openmessaging.storage.dledger.utils.DLedgerUtils;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.common.ThreadFactoryImpl;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
+
+public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler {
+
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
+    private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_"));
+    private BrokerController brokerController;
+    private DefaultMessageStore messageStore;
+    private DLedgerCommitLog dLedgerCommitLog;
+    private DLedgerServer dLegerServer;
+    public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) {
+        this.brokerController = brokerController;
+        this.messageStore = messageStore;
+        this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
+        this.dLegerServer = dLedgerCommitLog.getdLedgerServer();
+    }
+
+    @Override public void handle(long term, MemberState.Role role) {
+        Runnable runnable = new Runnable() {
+            @Override public void run() {
+                long start = System.currentTimeMillis();
+                try {
+                    boolean succ = true;
+                    log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole());
+                    switch (role) {
+                        case CANDIDATE:
+                            if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) {
+                                brokerController.changeToSlave(dLedgerCommitLog.getId());
+                            }
+                            break;
+                        case FOLLOWER:
+                            brokerController.changeToSlave(dLedgerCommitLog.getId());
+                            break;
+                        case LEADER:
+                            while (true) {
+                                if (!dLegerServer.getMemberState().isLeader()) {
+                                    succ = false;
+                                    break;
+                                }
+                                if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == -1) {
+                                    break;
+                                }
+                                if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex()
+                                    && messageStore.dispatchBehindBytes() == 0) {
+                                    break;
+                                }
+                                Thread.sleep(100);
+                            }
+                            if (succ) {
+                                messageStore.recoverTopicQueueTable();
+                                brokerController.changeToMaster(BrokerRole.SYNC_MASTER);
+                            }
+                            break;
+                        default:
+                            break;
+                    }
+                    log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start));
+                } catch (Throwable t) {
+                    log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t);
+                }
+            }
+        };
+        executorService.submit(runnable);
+    }
+
+    @Override public void startup() {
+
+    }
+
+    @Override public void shutdown() {
+        executorService.shutdown();
+    }
+}

broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java

@@ -152,7 +152,7 @@ public class BrokerOuterAPI {
                                 registerBrokerResultList.add(result);
                             }
 
-                            log.info("register broker to name server {} OK", namesrvAddr);
+                            log.info("register broker[{}]to name server {} OK", brokerId, namesrvAddr);
                         } catch (Exception e) {
                             log.warn("registerBroker Exception, {}", namesrvAddr, e);
                         } finally {

broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java

@@ -23,7 +23,6 @@ import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
 import org.apache.rocketmq.common.filter.ExpressionType;
-import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
 import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody;
@@ -36,6 +35,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
 import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.common.sysflag.TopicSysFlag;
 import org.apache.rocketmq.filter.FilterFactory;
+import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;

broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java

@@ -54,7 +54,7 @@ public class SlaveSynchronize {
 
     private void syncTopicConfig() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 TopicConfigSerializeWrapper topicWrapper =
                     this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);
@@ -78,7 +78,7 @@ public class SlaveSynchronize {
 
     private void syncConsumerOffset() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 ConsumerOffsetSerializeWrapper offsetWrapper =
                     this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak);
@@ -94,7 +94,7 @@ public class SlaveSynchronize {
 
     private void syncDelayOffset() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 String delayOffset =
                     this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak);
@@ -118,7 +118,7 @@ public class SlaveSynchronize {
 
     private void syncSubscriptionGroupConfig() {
         String masterAddrBak = this.masterAddr;
-        if (masterAddrBak != null) {
+        if (masterAddrBak != null  && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
             try {
                 SubscriptionGroupWrapper subscriptionWrapper =
                     this.brokerController.getBrokerOuterAPI()

broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java

@@ -22,36 +22,15 @@ import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 
-import java.util.concurrent.atomic.AtomicBoolean;
-
 public class TransactionalMessageCheckService extends ServiceThread {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME);
 
     private BrokerController brokerController;
 
-    private final AtomicBoolean started = new AtomicBoolean(false);
-
     public TransactionalMessageCheckService(BrokerController brokerController) {
         this.brokerController = brokerController;
     }
 
-    @Override
-    public void start() {
-        if (started.compareAndSet(false, true)) {
-            super.start();
-            this.brokerController.getTransactionalMessageService().open();
-        }
-    }
-
-    @Override
-    public void shutdown(boolean interrupt) {
-        if (started.compareAndSet(true, false)) {
-            super.shutdown(interrupt);
-            this.brokerController.getTransactionalMessageService().close();
-            this.brokerController.getTransactionalMessageCheckListener().shutDown();
-        }
-    }
-
     @Override
     public String getServiceName() {
         return TransactionalMessageCheckService.class.getSimpleName();

client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java

@@ -548,10 +548,10 @@ public class MQClientInstance {
                                 }
                             } catch (Exception e) {
                                 if (this.isBrokerInNameServer(addr)) {
-                                    log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr);
+                                    log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr, e);
                                 } else {
                                     log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName,
-                                        id, addr);
+                                        id, addr, e);
                                 }
                             }
                         }
@@ -1046,20 +1046,8 @@ public class MQClientInstance {
             if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) {
                 return this.brokerVersionTable.get(brokerName).get(brokerAddr);
             }
-        } else {
-            HeartbeatData heartbeatData = prepareHeartbeatData();
-            try {
-                int version = this.mQClientAPIImpl.sendHearbeat(brokerAddr, heartbeatData, 3000);
-                return version;
-            } catch (Exception e) {
-                if (this.isBrokerInNameServer(brokerAddr)) {
-                    log.info("send heart beat to broker[{} {}] failed", brokerName, brokerAddr);
-                } else {
-                    log.info("send heart beat to broker[{} {}] exception, because the broker not up, forget it", brokerName,
-                        brokerAddr);
-                }
-            }
         }
+        //To do need to fresh the version
         return 0;
     }
 

client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java

@@ -36,6 +36,7 @@ import org.apache.rocketmq.common.stats.StatsItemSet;
 import org.apache.rocketmq.remoting.exception.RemotingException;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Mock;
@@ -140,6 +141,7 @@ public class ConsumeMessageConcurrentlyServiceTest {
         pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet);
     }
 
+    @Ignore
     @Test
     public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception {
         final CountDownLatch countDownLatch = new CountDownLatch(1);

common/src/main/java/org/apache/rocketmq/common/ServiceThread.java

@@ -27,18 +27,29 @@ public abstract class ServiceThread implements Runnable {
 
     private static final long JOIN_TIME = 90 * 1000;
 
-    protected final Thread thread;
+    private Thread thread;
     protected final CountDownLatch2 waitPoint = new CountDownLatch2(1);
     protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false);
     protected volatile boolean stopped = false;
+    protected boolean isDaemon = false;
+
+    //Make it able to restart the thread
+    private final AtomicBoolean started = new AtomicBoolean(false);
 
     public ServiceThread() {
-        this.thread = new Thread(this, this.getServiceName());
+
     }
 
     public abstract String getServiceName();
 
     public void start() {
+        log.info("Try to start service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread);
+        if (!started.compareAndSet(false, true)) {
+            return;
+        }
+        stopped = false;
+        this.thread = new Thread(this, getServiceName());
+        this.thread.setDaemon(isDaemon);
         this.thread.start();
     }
 
@@ -47,6 +58,10 @@ public abstract class ServiceThread implements Runnable {
     }
 
     public void shutdown(final boolean interrupt) {
+        log.info("Try to shutdown service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread);
+        if (!started.compareAndSet(true, false)) {
+            return;
+        }
         this.stopped = true;
         log.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt);
 
@@ -75,11 +90,16 @@ public abstract class ServiceThread implements Runnable {
         return JOIN_TIME;
     }
 
+    @Deprecated
     public void stop() {
         this.stop(false);
     }
 
+    @Deprecated
     public void stop(final boolean interrupt) {
+        if (!started.get()) {
+            return;
+        }
         this.stopped = true;
         log.info("stop thread " + this.getServiceName() + " interrupt " + interrupt);
 
@@ -93,6 +113,9 @@ public abstract class ServiceThread implements Runnable {
     }
 
     public void makeStop() {
+        if (!started.get()) {
+            return;
+        }
         this.stopped = true;
         log.info("makestop thread " + this.getServiceName());
     }
@@ -128,4 +151,12 @@ public abstract class ServiceThread implements Runnable {
     public boolean isStopped() {
         return stopped;
     }
+
+    public boolean isDaemon() {
+        return isDaemon;
+    }
+
+    public void setDaemon(boolean daemon) {
+        isDaemon = daemon;
+    }
 }

common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java

@@ -41,6 +41,7 @@ public class MessageDecoder {
     public final static int MESSAGE_MAGIC_CODE = -626843481;
     public static final char NAME_VALUE_SEPARATOR = 1;
     public static final char PROPERTY_SEPARATOR = 2;
+    public static final int PHY_POS_POSITION =  4 + 4 + 4 + 4 + 4 + 8;
     public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE
         + 4 // 2 MAGICCODE
         + 4 // 3 BODYCRC

common/src/test/java/org/apache/rocketmq/common/protocol/GroupListTest.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol;
+
+import org.apache.rocketmq.common.protocol.body.GroupList;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.UUID;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Created by guoyao on 2019/2/18.
+ */
+public class GroupListTest {
+
+    @Test
+    public void testSetGet() throws Exception {
+        HashSet<String> fisrtUniqueSet=createUniqueNewSet();
+        HashSet<String> secondUniqueSet=createUniqueNewSet();
+        assertThat(fisrtUniqueSet).isNotEqualTo(secondUniqueSet);
+        GroupList gl=new GroupList();
+        gl.setGroupList(fisrtUniqueSet);
+        assertThat(gl.getGroupList()).isEqualTo(fisrtUniqueSet);
+        assertThat(gl.getGroupList()).isNotEqualTo(secondUniqueSet);
+        gl.setGroupList(secondUniqueSet);
+        assertThat(gl.getGroupList()).isNotEqualTo(fisrtUniqueSet);
+        assertThat(gl.getGroupList()).isEqualTo(secondUniqueSet);
+    }
+
+    private HashSet<String> createUniqueNewSet() {
+        HashSet<String> groups=new HashSet<String>();
+        groups.add(UUID.randomUUID().toString());
+        return groups;
+    }
+}

common/src/test/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfoTest.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.protocol.body;
+
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Properties;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import static org.apache.rocketmq.common.protocol.heartbeat.ConsumeType.CONSUME_ACTIVELY;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ConsumerRunningInfoTest {
+
+    private ConsumerRunningInfo consumerRunningInfo;
+
+    private TreeMap<String, ConsumerRunningInfo> criTable;
+
+    private MessageQueue messageQueue;
+
+    @Before
+    public void init() {
+        consumerRunningInfo = new ConsumerRunningInfo();
+        consumerRunningInfo.setJstack("test");
+
+        TreeMap<MessageQueue, ProcessQueueInfo> mqTable = new TreeMap<MessageQueue, ProcessQueueInfo>();
+        messageQueue = new MessageQueue("topicA","broker", 1);
+        mqTable.put(messageQueue, new ProcessQueueInfo());
+        consumerRunningInfo.setMqTable(mqTable);
+
+        TreeMap<String, ConsumeStatus> statusTable = new TreeMap<String, ConsumeStatus>();
+        statusTable.put("topicA", new ConsumeStatus());
+        consumerRunningInfo.setStatusTable(statusTable);
+
+        TreeSet<SubscriptionData> subscriptionSet = new TreeSet<SubscriptionData>();
+        subscriptionSet.add(new SubscriptionData());
+        consumerRunningInfo.setSubscriptionSet(subscriptionSet);
+
+        Properties properties = new Properties();
+        properties.put(ConsumerRunningInfo.PROP_CONSUME_TYPE, CONSUME_ACTIVELY);
+        properties.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, System.currentTimeMillis());
+        consumerRunningInfo.setProperties(properties);
+
+        criTable = new TreeMap<String, ConsumerRunningInfo>();
+        criTable.put("client_id", consumerRunningInfo);
+    }
+
+    @Test
+    public void testFromJson() {
+        String toJson = RemotingSerializable.toJson(consumerRunningInfo, true);
+        ConsumerRunningInfo fromJson = RemotingSerializable.fromJson(toJson, ConsumerRunningInfo.class);
+
+        assertThat(fromJson.getJstack()).isEqualTo("test");
+        assertThat(fromJson.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).isEqualTo(ConsumeType.CONSUME_ACTIVELY.name());
+
+        ConsumeStatus consumeStatus = fromJson.getStatusTable().get("topicA");
+        assertThat(consumeStatus).isExactlyInstanceOf(ConsumeStatus.class);
+
+        SubscriptionData subscription = fromJson.getSubscriptionSet().first();
+        assertThat(subscription).isExactlyInstanceOf(SubscriptionData.class);
+
+        ProcessQueueInfo processQueueInfo = fromJson.getMqTable().get(messageQueue);
+        assertThat(processQueueInfo).isExactlyInstanceOf(ProcessQueueInfo.class);
+    }
+
+    @Test
+    public void testAnalyzeRebalance(){
+        boolean result = ConsumerRunningInfo.analyzeRebalance(criTable);
+        assertThat(result).isTrue();
+    }
+
+    @Test
+    public void testAnalyzeProcessQueue(){
+        String result = ConsumerRunningInfo.analyzeProcessQueue("client_id", consumerRunningInfo);
+        assertThat(result).isEmpty();
+
+    }
+
+    @Test
+    public void testAnalyzeSubscription(){
+        boolean result = ConsumerRunningInfo.analyzeSubscription(criTable);
+        assertThat(result).isTrue();
+    }
+
+
+}

distribution/bin/dledger/fast-try.sh

@@ -0,0 +1,102 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+## Revise the base dir
+CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)"
+RMQ_DIR=$CURRENT_DIR/../..
+cd $RMQ_DIR
+
+function startNameserver() {
+    export JAVA_OPT_EXT=" -Xms512m -Xmx512m  "
+    nohup bin/mqnamesrv &
+}
+
+function startBroker() {
+    export JAVA_OPT_EXT=" -Xms1g -Xmx1g  "
+    conf_name=$1
+    nohup bin/mqbroker -c $conf_name &
+}
+
+function stopNameserver() {
+    PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}')
+    if [ ! -z "$PIDS" ]; then
+        kill -s TERM $PIDS
+    fi
+}
+
+function stopBroker() {
+    conf_name=$1
+    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    i=1
+    while [ ! -z "$PIDS" -a $i -lt 5 ]
+    do
+        echo "Waiting to kill ..."
+        kill -s TERM $PIDS
+        ((i=$i+1))
+        sleep 2
+        PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    done
+    PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}')
+    if [ ! -z "$PIDS" ]; then
+        kill -9 $PIDS
+    fi
+}
+
+function stopAll() {
+    ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill
+    stopNameserver
+    stopBroker ./conf/dledger/broker-n0.conf
+    stopBroker ./conf/dledger/broker-n1.conf
+    stopBroker ./conf/dledger/broker-n2.conf
+}
+
+function startAll() {
+    startNameserver
+    startBroker ./conf/dledger/broker-n0.conf
+    startBroker ./conf/dledger/broker-n1.conf
+    startBroker ./conf/dledger/broker-n2.conf
+}
+
+function checkConf() {
+    if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then
+        echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists"
+        exit -1
+    fi 
+}
+
+
+
+## Main
+if [ $# -lt 1 ]; then
+    echo "Usage: sh $0 start|stop"
+    exit -1
+fi
+action=$1
+checkConf
+case $action in
+    "start")
+        startAll
+        exit
+        ;;
+    "stop")
+        stopAll
+        ;;
+    *)
+        echo "Usage: sh $0 start|stop"
+        ;;
+esac
+

distribution/conf/dledger/broker-n0.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30911
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node00
+storePathCommitLog=/tmp/rmqstore/node00/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n0
+sendMessageThreadPoolNums=16

distribution/conf/dledger/broker-n1.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30921
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node01
+storePathCommitLog=/tmp/rmqstore/node01/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n1
+sendMessageThreadPoolNums=16

distribution/conf/dledger/broker-n2.conf

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30931
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node02
+storePathCommitLog=/tmp/rmqstore/node02/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n2
+sendMessageThreadPoolNums=16

distribution/release.xml

@@ -40,7 +40,7 @@
 
         <fileSet>
             <includes>
-                <include>bin/*</include>
+                <include>bin/**</include>
             </includes>
             <fileMode>0755</fileMode>
         </fileSet>

docs/cn/RocketMQ_Example.md

@@ -639,9 +639,9 @@ consumer.start();
 
 事务消息共有三种状态，提交状态、回滚状态、中间状态：
 
-1. TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。
-2. TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。
-3. TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。
+- TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。
+- TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。
+- TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。
 
 ### 6.1 发送事务消息样例
 

docs/cn/concept.md

@@ -1,9 +1,7 @@
 # 基本概念
 ## 消息模型（Message Model）
 
-<img src="./images/message_model_1.jpg" style="width: 560px"/>
-
-RocketMQ 消息模型如图1所示，主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。
+RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。
 ##  消息生产者（Producer）
  负责生产消息，一般由业务系统负责生产消息。一个消息生产者会把业务应用系统里产生的消息发送到broker服务器。RocketMQ提供多种发送方式，同步发送、异步发送、顺序发送、单向发送。同步和异步方式均需要Broker返回确认信息，单向发送不需要。
 ## 消息消费者（Consumer）

docs/cn/dledger/deploy_guide.md

@@ -0,0 +1,76 @@
+## 前言
+该文档主要介绍如何部署自动容灾切换的 RocketMQ-on-DLedger Group。
+
+RocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。  
+RocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。  
+RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。  
+
+## 1. 新集群部署
+
+#### 1.1 编写配置
+每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。  
+编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。  
+关键配置介绍：  
+
+| name | 含义 | 举例 |
+| --- | --- | --- |
+| enableDLegerCommitLog | 是否启动 DLedger  | true |
+| dLegerGroup | DLedger Raft Group的名字，建议和 brokerName 保持一致 | RaftNode00 |
+| dLegerPeers | DLedger Group 内各节点的端口信息，同一个 Group 内的各个节点配置必须要保证一致 | n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 |
+| dLegerSelfId | 节点 id, 必须属于 dLegerPeers 中的一个；同 Group 内各个节点要唯一 | n0 |
+| sendMessageThreadPoolNums | 发送线程个数，建议配置成 Cpu 核数 | 16 |
+
+这里贴出 conf/dledger/broker-n0.conf 的配置举例。  
+
+```
+brokerClusterName = RaftCluster
+brokerName=RaftNode00
+listenPort=30911
+namesrvAddr=127.0.0.1:9876
+storePathRootDir=/tmp/rmqstore/node00
+storePathCommitLog=/tmp/rmqstore/node00/commitlog
+enableDLegerCommitLog=true
+dLegerGroup=RaftNode00
+dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913
+## must be unique
+dLegerSelfId=n0
+sendMessageThreadPoolNums=16
+```
+
+### 1.2 启动 Broker
+
+与老版本的启动方式一致。
+
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & `  
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & `  
+`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & `  
+
+
+## 2. 旧集群升级
+
+如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。  
+如果旧集群采用 Master-Slave 方式部署，则每个 Master-Slave 组都需要转换成一个 RocketMQ-on-DLedger Group。
+
+### 2.1 杀掉旧的 Broker
+
+可以通过 kill 命令来完成，也可以调用 `bin/mqshutdown broker`。
+
+### 2.2 检查旧的 Commitlog
+
+RocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。  
+如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。  
+
+虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。  
+所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。  
+   
+在 3 台机器准备好了之后，旧 Commitlog 文件也保证一致之后，就可以开始走下一步修改配置了。
+
+### 2.3 修改配置
+
+参考新集群部署。
+
+### 2.4 重新启动 Broker 
+
+参考新集群部署。
+
+

docs/cn/dledger/quick_start.md

@@ -0,0 +1,61 @@
+### 前言
+该文档主要介绍如何快速构建和部署基于 DLedger 的可以自动容灾切换的 RocketMQ 集群。
+
+详细的新集群部署和旧集群升级指南请参考 [部署指南](deploy_guide.md)。
+
+### 1. 源码构建
+构建分为两个部分，需要先构建 DLedger，然后 构建 RocketMQ
+
+#### 1.1 构建 DLedger
+
+`git clone https://github.com/openmessaging/openmessaging-storage-dledger.git`
+
+`cd openmessaging-storage-dledger`
+
+`mvn clean install -DskipTests`
+
+#### 1.2 构建 RocketMQ
+
+`git clone https://github.com/apache/rocketmq.git`
+
+`cd rocketmq`
+
+`git checkout -b store_with_dledger origin/store_with_dledger`
+
+`mvn -Prelease-all -DskipTests clean install -U`
+
+### 2. 快速部署
+
+在构建成功后
+
+`cd distribution/target/apache-rocketmq`
+
+`sh bin/dledger/fast-try.sh start`
+
+如果上面的步骤执行成功，可以通过 mqadmin 运维命令查看集群状态。
+
+`sh bin/mqadmin clusterList -n 127.0.0.1:9876`
+
+顺利的话，会看到如下内容：
+
+![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa)
+
+（BID 为 0 的表示 Master，其余都是 Follower）
+
+启动成功，现在可以向集群收发消息，并进行容灾切换测试了。
+
+关闭快速集群，可以执行：
+
+`sh bin/dledger/fast-try.sh stop`
+
+快速部署，默认配置在 conf/dledger 里面，默认的存储路径在 /tmp/rmqstore。
+
+
+### 3. 容灾切换
+
+部署成功，杀掉 Leader 之后（在上面的例子中，杀掉端口 30931 所在的进程），等待约 10s 左右，用 clusterList 命令查看集群，就会发现 Leader 切换到另一个节点了。
+
+
+
+
+

docs/cn/operation.md

@@ -1,4 +1,4 @@
-## 运维管理(operation）
+## 运维管理
 
 ### 1   集群搭建
 

docs/en/CLITools.md

@@ -2,12 +2,12 @@
 
 Before introducing the mqadmin management tool, the following points need to be declared:
 
-1. The way of executing a command is:./mqadmin {command} {args}
-2. Almost all commands need to attach the -n option to represent the nameServer address, formatted as ip:port;
-3. Almost all commands can get help information with the -h option;
-4. If the broker address -b option and clusterName -c option are both configured with specific values, the command execution will select the broker address specified by -b option. The value of the -b option can only be configured with a single address. The format is ip:port. The default port value is 10911. If the value of the -b option is not configured, the command will be applied to all brokers in the entire cluster.
-5. You can see many commands under tools, but not all commands can be used, only the commands initialized in MQAdminStartup can be used, you can also modify this class, add or customize commands;
-6. Due to the issue of version update, a small number of commands may not be updated in time, please read the related command source code to eliminate and resolve the error.
+- The way of executing a command is:./mqadmin {command} {args}
+- Almost all commands need to attach the -n option to represent the nameServer address, formatted as ip:port;
+- Almost all commands can get help information with the -h option;
+- If the broker address -b option and clusterName -c option are both configured with specific values, the command execution will select the broker address specified by -b option. The value of the -b option can only be configured with a single address. The format is ip:port. The default port value is 10911. If the value of the -b option is not configured, the command will be applied to all brokers in the entire cluster.
+- You can see many commands under tools, but not all commands can be used, only the commands initialized in MQAdminStartup can be used, you can also modify this class, add or customize commands;
+- Due to the issue of version update, a small number of commands may not be updated in time, please read the related command source code to eliminate and resolve the error.
 
 ## 1 Topic related command instructions
 
@@ -262,7 +262,7 @@ Before introducing the mqadmin management tool, the following points need to be
   height:244.0pt;border-top:none;width:133pt'><span
   style='mso-spacerun:yes'> </span>clusterList</td>
   <td rowspan=4 class=xl70 width=175 style='border-bottom:1.0pt;
-  border-top:none;width:131pt'>View cluster information, cluster, brokerName, brokerId, TPS, and so on.</td>
+  border-top:none;width:131pt'>View cluster information, cluster, brokerName, brokerId, TPS, and so on</td>
   <td class=xl65 width=177 style='width:133pt'>-m</td>
   <td class=xl66 width=185 style='width:139pt'>Print more information (add print to # InTotalYest,
   #OutTotalYest, #InTotalToday ,#OutTotalToday)</td>
@@ -283,7 +283,7 @@ Before introducing the mqadmin management tool, the following points need to be
   <td rowspan=8 height=391 class=xl67 width=177 style='border-bottom:1.0pt;
   height:292.0pt;border-top:none;width:133pt'>clusterRT</td>
   <td rowspan=8 class=xl70 width=175 style='border-bottom:1.0pt;
-  border-top:none;width:131pt'>Send message to detect each broker RT of the cluster.the message send to ${BrokerName} Topic。</td>
+  border-top:none;width:131pt'>Send message to detect each broker RT of the cluster.the message send to ${BrokerName} Topic</td>
   <td class=xl65 width=177 style='width:133pt'>-a</td>
   <td class=xl66 width=185 style='width:139pt'>amount，total number per probe，RT = Total time/amount</td>
  </tr>

docs/en/Concept.md

@@ -2,40 +2,40 @@
 
 ## 1 Message Model
 
-The RocketMQ message model is mainly composed of Producer, Broker and Consumer. The Producer is responsible for producing messages, the Consumer is responsible for consuming messages, and the Broker is responsible for storing messages.     
-The Broker corresponds to one server during actual deployment, and each Broker can store messages from multiple topics, and messages from each Topic can be stored in a different Broker by sharding strategy.     
-The Message Queue is used to store physical offsets of messages, and the Message addresses in multiple Topic are stored in multiple Message queues. The Consumer Group consists of multiple Consumer instances.    
+RocketMQ message model is mainly composed of Producer, Broker and Consumer. The producer is responsible for producing messages and the consumer is for consuming messages, while the broker stores messages.     
+The broker is an independent server during actual deployment, and each broker can store messages from multiple topics. Even messages from the same topic can be stored in the different brokers by sharding strategy.     
+The message queue is used to store physical offsets of messages, and the message addresses are stored in seperate queues. The consumer group consists of multiple consumer instances.    
 ##  2 Producer    
-The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the business application systems to brokers. The RocketMQ provides multiple paradigms of sending: synchronous, asynchronous,sequential and one-way. Both synchronous and asynchronous methods require the Broker to return confirmation information, while one-way sending is not required.
+The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the systems to brokers. RocketMQ provides multiple paradigms of sending: synchronous, asynchronous, sequential and one-way. Both synchronous and asynchronous methods require the confirmation information return from the Broker, but one-way method does not require it.
 ## 3 Consumer
- The Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The Consumer pulls messages from brokers and feeds them into application. In perspective of user application, two types of consumers are provided:Pull Consumer and Push Consumer.
+The Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The consumer pulls messages from brokers and feeds them into application. From the perspective of user, two types of consumers are provided: pull consumer and push consumer.
 ## 4 Topic
-The Topic refers to a collection of one kind message. Each topic contains several messages and one message can only belong to one topic. The Topic is the basic unit of RocketMQ for message subscription.
+The Topic refers to a collection of one kind of message. Each topic contains several messages and one message can only belong to one topic. The topic is the basic unit of RocketMQ for message subscription.
 ## 5 Broker Server
-As the role of the transfer station, the Broker Server stores, and forwards messages. In the RocketMQ system, the Broker Server is responsible for receiving messages sent from producers, storing them and preparing to handle pull requests. It also stores message related meta data, including consumer groups, consuming progress, topics and queues info.
+As the role of the transfer station, the Broker Server stores and forwards messages. In RocketMQ, the broker server is responsible for receiving messages sent from producers, storing them and preparing to handle pull requests. It also stores the related message meta data, including consumer groups, consuming progress, topics, queues info and so on.
 ## 6 Name Server
-The Name Server serves as the provider of routing service. The Producer or Consumer can find the list of Broker IP addresses corresponding each topic through name server. Multiple Name Servers can be deployed as a cluster, but are independent of each other and do not exchange information.
+The Name Server serves as the provider of routing service. The producer or the consumer can find the list of broker IP addresses for each topic through name server. Multiple name servers can be deployed in one cluster, but they are independent of each other and do not exchange information.
 ## 7 Pull Consumer
- A type of Consumer. Applications are usually pulls messages from brokers by actively calling the Consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once batches of messages are pulled, user application initiates consuming process.                    
+A type of Consumer, the application pulls messages from brokers by actively invoking the consumer pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process.                    
 ## 8 Push Consumer
- A type of Consumer. Under this mode, after the Broker receives the data, it will actively push it to the consumer, which is generally of high real-time performance.                       
+A type of Consumer. Under this high real-time performance mode, it will push the message to the consumer actively when the Broker receives the data.                       
 ## 9 Producer Group
-  A collection of the same type of Producer, which sends the same type of messages with consistent logic. If a transaction message is sent and the original producer crashes after sending, the Broker Server will contacts other Producer in the same Producer group to commit or rollback the transactional message.
+A collection of the same type of Producer, which sends the same type of messages with consistent logic. If a transaction message is sent and the original producer crashes after sending, the broker server will contact other producers in the same producer group to commit or rollback the transactional message.
 ## 10 Consumer Group
-  A collection of the same type of Consumer, which sends the same type of messages with consistent logic. The Consumer Group makes load-balance and fault-tolerance super easy in terms of message consuming.
-Warning: consumer instances of one consumer group must have exactly the same topic subscription(s).   
+A collection of the same type of Consumer, which sends the same type of messages with consistent logic. The consumer group makes load-balance and fault-tolerance super easy in terms of message consuming.
+Warning: consumer instances of one consumer group must have exactly the same topic subscription(s).   
 
-RocketMQ supports two type of consumption mode:Clustering and Broadcasting.
+RocketMQ supports two types of consumption mode:Clustering and Broadcasting.
 ## 11 Consumption Mode - Clustering
-Under the Clustering mode, all messages from one topic will be delivered to all consumer instances averagely as much as possible. That is, one message can be consumed by only one consumer instance.
+Under the Clustering mode, all the messages from one topic will be delivered to all the consumers instances averagely as much as possible. That is, one message can be consumed by only one consumer instance.
 ## 12 Consumption Mode - Broadcasting
-Under the Broadcasting mode, each Consumer instance of the same Consumer Group receives every message that published to the corresponding topic.
+Under the Broadcasting mode, each consumer instance of the same consumer group receives every message published to the corresponding topic.
 ## 13 Normal Ordered Message
-Under the Normal Ordered Message mode, the messages received by consumers from the same ConsumeQueue are sequential, while the messages received from different message queues may be non-sequential.
+Under the Normal Ordered Message mode, the messages received by consumers from the same ConsumeQueue are sequential, but the messages received from the different message queues may be non-sequential.
 ## 14 Strictly Ordered Message
-Under the Strictly Ordered Message mode, all messages received by the consumers from the same topic are sequential, as the order they were stored.
+Under the Strictly Ordered Message mode, all messages received by the consumers from the same topic are sequential as the order they are stored.
 ## 15 Message
-The physical carrier of information transmitted by a messaging system, the smallest unit of production and consumption data, each message must belong to a topic.
-Each Message in RocketMQ has a unique Message ID and can carry a key, generally used to store business-related value. The system provides the function to query messages by Message ID and Key.
+The physical carrier of information transmitted by a messaging system, the smallest unit of production and consumption data, each message must belong to one topic.
+Each Message in RocketMQ has a unique message id and can carry a key used to store business-related value. The system has the function to query messages by its id or key.
 ## 16 Tag
- Flags set for messages to distinguish different types of messages under the same topic, functioning as a "sub-topic". Messages from the same business unit can set different tags under the same topic in terms of different business purposes. The Tag can effectively maintain the clarity and consistency of the code and optimize the query system provided by RocketMQ. The Consumer can realize different "sub-topic" by using Tag, so as to achieve better expansibility.
+Flags set for messages to distinguish different types of messages under the same topic, functioning as a "sub-topic". Messages from the same business unit can set different tags under the same topic in terms of different business purposes. The tag can effectively maintain the clarity and consistency of the code and optimize the query system provided by RocketMQ. The consumer can realize different "sub-topic" by using tag in order to achieve better expansibility.

docs/en/Configuration_Client.md

@@ -2,7 +2,7 @@
 
   Relative to RocketMQ's Broker cluster, producers and consumers are client. In this section, it mainly describes the common behavior configuration of producers and consumers.
 ​ 
-### Client Addressing mode
+### 1 Client Addressing mode
 
 ```RocketMQ``` can let client find the ```Name Server```, and then find the ```Broker```by the ```Name Server```. Followings show a variety of configurations, and priority level from highly to lower, the highly priority configurations can override the lower priority configurations.
 
@@ -36,11 +36,11 @@ By default, the client accesses the HTTP server every 2 minutes, and update the
 ```
 HTTP static server addressing is recommended, because it is simple client deployment, and the Name Server cluster can be upgraded hot.
 
-### Client Configuration
+### 2 Client Configuration
 
 ```DefaultMQProducer```,```TransactionMQProducer```,```DefaultMQPushConsumer```,```DefaultMQPullConsumer``` all extends the ```ClientConfig``` Class, ```ClientConfig``` as the client common configuration class. Client configuration style like getXXX,setXXX, each of the parameters can config by spring and also config their in the code. Such as the ```namesrvAddr``` parameter: ```producer.setNamesrvAddr("192.168.0.1:9876")```, same with the other parameters.
 
-#### Client Common Configuration
+#### 2.1 Client Common Configuration
 
 | Pamater Name                        | Default Value  | Description                                                         |
 | ----------------------------- | ------- | ------------------------------------------------------------ |
@@ -52,7 +52,7 @@ HTTP static server addressing is recommended, because it is simple client deploy
 | heartbeatBrokerInterval       | 30000   | The heartbeat interval, in milliseconds, is sent to the Broker                         |
 | persistConsumerOffsetInterval | 5000    | The persistent Consumer consumes the progress interval in milliseconds         |
 
-#### Producer Configuration
+#### 2.2 Producer Configuration
 
 | Pamater Name                       | Default Value          | Description                                                        |
 | -------------------------------- | ---------------- | ------------------------------------------------------------ |
@@ -70,7 +70,7 @@ HTTP static server addressing is recommended, because it is simple client deploy
 | checkRequestHoldMax              | 2000             | Producer local buffer request queue size when Broker look back Producer transaction status                     |
 | RPCHook                          | null             | This parameter is passed in when the Producer is creating, including the pre-processing before the message sending and the processing after the message response. The user can do some security control or other operations in the first interface.|
 
-#### PushConsumer Configuration
+#### 2.3 PushConsumer Configuration
 
 | Pamater Name                         | Default Value                      | Description                                                         |
 | ---------------------------- | ----------------------------- | ------------------------------------------------------------ |
@@ -91,7 +91,7 @@ HTTP static server addressing is recommended, because it is simple client deploy
 | consumeMessageBatchMaxSize   | 1                             | Batch consume message                                 |
 | pullBatchSize                | 32                            | Batch pull message                                 |
 
-#### PullConsumer Configuration
+#### 2.4 PullConsumer Configuration
 
 | Pamater Name                     | Default Value                 | Description                                                         |
 | -------------------------------- | ----------------------------- | ------------------------------------------------------------ |
@@ -105,7 +105,7 @@ HTTP static server addressing is recommended, because it is simple client deploy
 | registerTopics                   |                               | Collection of registered topics                                              |
 | allocateMessageQueueStrategy     | AllocateMessageQueueAveragely | Implements strategy about Rebalance algorithm                                     |
 
-#### Message Data Structure
+#### 2.5 Message Data Structure
 
 | Field Name         | Default Value  | Description                                                         |
 | -------------- | ------ | ------------------------------------------------------------ |

docs/en/Configuration_System.md

@@ -1,13 +1,17 @@
-# ** The system configuration** #
+# The system configuration
 
 This section focuses on the configuration of the system (JVM/OS)
 
 ## **1 JVM Options** ##
 
-The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this:
+The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows:
 
     -server -Xms8g -Xmx8g -Xmn4g
 
+Direct ByteBuffer memory size setting. Full GC will be triggered when the Direct ByteBuffer up to the specified size:
+
+    -XX:MaxDirectMemorySize=15g
+
 If you don’t care about the boot time of RocketMQ broker, pre-touch the Java heap to make sure that every page will be allocated during JVM initialization is a better choice. Those who don’t care about the boot time can enable it:
     
     -XX:+AlwaysPreTouch
@@ -59,7 +63,7 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic
 
 
 
-- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend set 655350 for file descriptors.
+- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend setting  655350 for file descriptors.
 
 
 

docs/en/Deployment.md

@@ -1,10 +1,8 @@
-# Deployment Architectures and Setup Steps
-
-## Cluster Setup
+# Installation Guides
 
 ### 1 Single Master mode
 
-This is the simplest, but also the riskiest, mode that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build.
+This is the simplest but also the riskiest mode, that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build.
 
 **1）Start NameServer**
 
@@ -69,7 +67,7 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker
 ...
 ```
 
-The boot command shown above is used in the case of a single NameServer.For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.161.2: 9876.
+The boot command shown above is used in the case of a single NameServer. For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.161.2: 9876.
 
 ### 3 Multiple Master And Multiple Slave Mode-Asynchronous replication
 

docs/en/Design_Filter.md

@@ -5,6 +5,6 @@ RocketMQ - a distributed message queue, is different with all other MQ middlewar
 
 The RocketMQ has two mainly filter types:
 
-* Tag filtering: Consumer can specify not only the message topic but also the message tag values, when subscribing. Multiple tag values need to be separated by '||'. When consumer subscribing a message, it builds the subscription request into a `SubscriptionData` object and sends a Pull message request to the Broker side. Before the Broker side reads data from the RocketMQ file storage layer - Store, it will construct a `MessageFilter` using the `SubscriptionData` object and then pass it to the Store. Store get a record from `ConsumeQueue`, and it will filter the message by the saved tag hashcode, it is unable to filter the messages exactly in the server side because of only the hashcode will be used when filtering, Therefore, after the Consumer pulls the message, it also needs to compare the original tag string of the message. If the original tag string is not same with the expected, the message will be ignored.
+* Tag filtering: Consumer can specify not only the message topic but also the message tag values, when subscribing. Multiple tag values need to be separated by '||'. When consumer subscribing a message, it builds the subscription request into a `SubscriptionData` object and sends a pull message request to the Broker side. Before the Broker side reads data from the RocketMQ file storage layer - Store, it will construct a `MessageFilter` using the `SubscriptionData` object and then pass it to the Store. Store get a record from `ConsumeQueue`, and it will filter the message by the saved tag hashcode, it is unable to filter the messages exactly in the server side because of only the hashcode will be used when filtering, Therefore, after the Consumer pulls the message, it also needs to compare the original tag string of the message. If the original tag string is not same with the expected, the message will be ignored.
 
-* SQL92 filtering: This filter behavior is almost same with the above Tag filtering method. The only difference is on the way how Store works. The rocketmq-filter module is responsible for the construction and execution of the real SQL expression. Executing an SQL expression every time a filter is executed affects efficiency, so RocketMQ uses BloomFilter to avoid doing it every time. The expression context of SQL92 is a property of the message.
+* SQL92 filtering: This filter behavior is almost same with the above `Tag filtering` method. The only difference is on the way how Store works. The rocketmq-filter module is responsible for the construction and execution of the real SQL expression. Executing an SQL expression every time a filter is executed affects efficiency, so RocketMQ uses BloomFilter to avoid doing it every time. The expression context of SQL92 is a property of the message.

docs/en/Design_LoadBlancing.md

@@ -1,11 +1,11 @@
-## 4 Load Balancing
+## Load Balancing
 Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages.
 
-### 4.1 Producer Load Balancing
+### 1 Producer Load Balancing
 When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo  to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class.
 Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending.
 
-### 4.2 Consumer Load Balancing
+### 2 Consumer Load Balancing
 In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side.
 
  1， Heartbeat Packet Sending on Consumer side

docs/en/Design_Remoting.md

@@ -1,17 +1,17 @@
-## 2 Communication Mechanism
+
 RocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows:
 (1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds.
-(2) When message producer Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default.
-(3) Message producer Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message.
-(4) After message consumer Consumer get the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them.
+(2) When message Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default.
+(3) Message Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message.
+(4) After message Consumer gets the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them.
 
-From 1) ~ 3 above, we can see that both Producer, Broker and NameServer communicate with each other(only part of MQ communication is mentioned here), so how to design a good network communication module is very important in MQ. It will determine the overall messaging capability and final performance of the RocketMQ cluster.
+From 1) ~ 3) above, we can see that both Producer, Broker and NameServer communicate with each other(only part of MQ communication is mentioned here), so how to design a good network communication module is very important in MQ. It will determine the overall messaging capability and final performance of the RocketMQ cluster.
 
 rocketmq-remoting module is the module responsible for network communication in RocketMQ message queue. It is relied on and referenced by almost all other modules (such as rocketmq-client,rocketmq-broker,rocketmq-namesrv) that need network communication. In order to realize the efficient data request and reception between the client and the server, the RocketMQ message queue defines the communication protocol and extends the communication module on the basis of Netty.
 
-### 2.1 Remoting Communication Class Structure
+### 1 Remoting Communication Class Structure
 ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_3.png)
-### 2.2 Protocol Design and Codec
+### 2 Protocol Design and Code
 When a message is sent between Client and Server, a protocol convention is needed for the message sent, so it is necessary to customize the message protocol of RocketMQ. At the same time, in order to efficiently transmit messages and read the received messages, it is necessary to encode and decode the messages. In RocketMQ, the RemotingCommand class encapsulates all data content in the process of message transmission, which includes not only all data structures, but also encoding and decoding operations.
 
 Header field | Type | Request desc | Response desc
@@ -33,10 +33,10 @@ From the above figure, the transport content can be divided into four parts:
 (3) Header data: serialized header data;
 
 (4) Message body data: binary byte data content of message body;
-#### 2.3 Message Communication Mode and Procedure
+### 3 Message Communication Mode and Procedure
 There are three main ways to support communication in RocketMQ message queue: synchronous (sync), asynchronous (async), one-way (oneway). The "one-way" communication mode is relatively simple and is generally used in sending heartbeat packets without paying attention to its Response. Here, mainly introduce the asynchronous communication flow of RocketMQ.
 ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_5.png)
-#### 2.4 Reactor Multithread Design
+### 4 Reactor Multithread Design
 The RPC communication of RocketMQ uses Netty component as the underlying communication library, and also follows the Reactor multithread model. At the same time, some extensions and optimizations are made on it.
 ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_6.png)
 Above block diagram can roughly understand the Reactor multi-thread model of NettyRemotingServer in RocketMQ. A Reactor main thread (eventLoopGroupBoss, is 1 above) is responsible for listening to TCP network connection requests, establishing connections, creating SocketChannel, and registering on selector. The source code of RocketMQ automatically selects NIO and Epoll according to the type of OS. Then listen to real network data. After you get the network data, you throw it to the Worker thread pool (eventLoopGroupSelector, is the "N" above, the default is 3 in the source code). You need to do SSL verification, codec, idle check, network connection management before you really execute the business logic. These tasks to defaultEventExecutorGroup (that is, "M1" above, the default set to 8 in the source code) to do. The processing business operations are executed in the business thread pool. According to the RomotingCommand business request code, the corresponding processor is found in the processorTable local cache variable and encapsulated into the task, and then submitted to the corresponding business processor processing thread pool for execution (sendMessageExecutor,). Take sending a message, for example, the "M2" above. The thread pool continues to increase in several steps from entry to business logic, which is related to the complexity of each step. The more complex the thread pool is, the wider the concurrent channel is required.

docs/en/Design_Store.md

@@ -15,13 +15,13 @@ The message storage architecture diagram consists of 3 files related to message
 
 
 * `CommitLog`：The `CommitLog` file stores message body and metadata sent by producer, and the message content is not fixed length. The default size of one `CommitLog` file is 1G, the length of the file name is 20 digits, the left side is zero padded, and the remaining is the starting offset. For example, `00000000000000000000` represents the first file, the starting offset is 0, and the file size is 1G=1073741824, when the first `CommitLog` file is full, the second `CommitLog` file is `00000000001073741824`, the starting offset is 1073741824, and so on. The message is mainly appended to the log file sequentially. When one `CommitLog` file is full, the next will be written.
-* `ConsumeQueue`: The `ConsumeQueue` is used to improve the performance of message consumption. Since RocketMQ uses topic-based subscription mode, message consumption is specific to the topic. Traversing the commitlog file to retrieve messages of one topic is very inefficient. The consumer can find the messages to be consumed according to the `ConsumeQueue`. The `ConsumeQueue`(logic consume queue) as an index of the consuming message stores the starting physical offset `offset` in `CommitLog` of the specified topic, the message size `size` and the hash code of the message tag. The `ConsumeQueue` file can be regarded as a topic-based `CommitLog` index file, so the consumequeue folder is organized as follows: `topic/queue/file` three-layer organization structure, the specific storage path is `$HOME/store/consumequeue/{topic}/{queueId }/{fileName}`. The consumequeue file uses a fixed-length design, each entry occupies 20 bytes, which is an 8-byte commitlog physical offset, a 4-byte message length, and an 8-byte tag hashcode. One consumequeue file consists of 30W entries, each entry can be randomly accessed like an array, each `ConsumeQueue` file's size is about 5.72M.
+* `ConsumeQueue`: The `ConsumeQueue` is used to improve the performance of message consumption. Since RocketMQ uses topic-based subscription mode, message consumption is specific to the topic. Traversing the commitlog file to retrieve messages of one topic is very inefficient. The consumer can find the messages to be consumed according to the `ConsumeQueue`. The `ConsumeQueue`(logic consume queue) as an index of the consuming message stores the starting physical offset `offset` in `CommitLog` of the specified topic, the message size `size` and the hash code of the message tag. The `ConsumeQueue` file can be regarded as a topic-based `CommitLog` index file, so the consumequeue folder is organized as follows: `topic/queue/file` three-layer organization structure, the specific storage path is `$HOME/store/consumequeue/{topic}/{queueId }/{fileName}`. The consumequeue file uses a fixed-length design, each entry occupies 20 bytes, which is an 8-byte commitlog physical offset, a 4-byte message length, and an 8-byte tag hashcode. One consumequeue file consists of 0.3 million entries, each entry can be randomly accessed like an array, each `ConsumeQueue` file's size is about 5.72MB.
 * `IndexFile`: The `IndexFile` provides a way to query messages by key or time interval. The path of the `IndexFile` is `$HOME/store/index/${fileName}`, the file name `fileName` is named after the timestamp when it was created. One IndexFile's size is about 400M, and it can store 2000W indexes. The underlying storage of `IndexFile` is designed to implement the `HashMap` structure in the file system, so RocketMQ's index file is a hash index.
 
 
 From the above architecture of the RocketMQ message storage, we can see RocketMQ uses a hybrid storage structure, that is, all the queues in an instance of the broker share a single log file `CommitLog` to store messages. RocketMQ's hybrid storage structure(messages of multiple topics are stored in one CommitLog) uses a separate storage structure for the data and index parts for Producer and Consumer respectively. The Producer sends the message to the Broker, then the Broker persists the message to the CommitLog file synchronously or asynchronously. As long as the message is persisted to the CommitLog on the disk, the message sent by the Producer will not be lost. Because of this, Consumer will definitely have the opportunity to consume this message. When no message can be pulled, the consumer can wait for the next pull. And the server also supports the long polling mode: if a pull request pulls no messages, the Broker can wait for 30 seconds, as long as new message arrives in this interval, it will be returned directly to the consumer. Here, RocketMQ's specific approach is using Broker's background service thread `ReputMessageService` to continuously dispatch requests and asynchronously build ConsumeQueue (Logical Queue) and IndexFile data.
 
-## 2 PageCache and memory map
+## 2 PageCache and Memory Map
 
 PageCache is a cache of files by the operating system to speed up the reading and writing of files. In general, the speed of sequential read and write files is almost the same as the speed of read and write memory. The main reason is that the OS uses a portion of the memory as PageCache to optimize the performance of the read and write operations. For data writing, the OS will first write to the Cache, and then the `pdflush` kernel thread asynchronously flush the data in the Cache to the physical disk. For data reading, if it can not hit the page cache when reading a file at a time, the OS will read the file from the physical disk and prefetch the data files of other neighboring blocks sequentially.
 

docs/en/Design_Trancation.md

@@ -7,20 +7,20 @@ Apache RocketMQ supports distributed transaction message from version 4.3.0. Roc
 ### 1.1 The Process of RocketMQ Transaction Message
 The picture above shows the overall architecture of transaction message, including the sending of message(commit-request phase), the sending of commit/rollback(commit phase) and the compensation process.
 
-1. The sending of message and Commit/Rollback.  
+1 The sending of message and Commit/Rollback.  
   (1) Sending the message(named Half message in RocketMQ)  
   (2) The server responds the writing result(success or failure) of Half message.  
   (3) Handle local transaction according to the result(local transaction won't be executed when the result is failure).  
   (4) Sending Commit/Rollback to broker according to the result of local transaction(Commit will generate message index and make the message visible to consumers).
 
-2. Compensation process  
+2 Compensation process  
   (1) For a transaction message without a Commit/Rollback (means the message in the pending status), a "back-check" request is initiated from the broker.  
   (2) The Producer receives the "back-check" request and checks the status of the local transaction corresponding to the "back-check" message.  
   (3) Redo Commit or Rollback based on local transaction status.
 The compensation phase is used to resolve the timeout or failure case of the message Commit or Rollback.
 
 ### 1.2 The design of RocketMQ Transaction Message
-1. Transaction message is invisible to users in first phase(commit-request phase)   
+1 Transaction message is invisible to users in first phase(commit-request phase)   
   
   Upon on the main process of transaction message, the message of first phase is invisible to the user. This is also the biggest difference from normal message. So how do we write the message while making it invisible to the user? And below is the solution of RocketMQ: if the message is a Half message, the topic and queueId of the original message will be backed up, and then changes the topic to RMQ_SYS_TRANS_HALF_TOPIC. Since the consumer group does not subscribe to the topic, the consumer cannot consume the Half message. Then RocketMQ starts a timing task, pulls the message for RMQ_SYS_TRANS_HALF_TOPIC, obtains a channel according to producer group and sends a back-check to query local transaction status, and decide whether to submit or roll back the message according to the status.  
   
@@ -30,21 +30,21 @@ The compensation phase is used to resolve the timeout or failure case of the mes
 
   The specific implementation strategy of RocketMQ is: if the transaction message is written, topic and queueId of the message are replaced, and the original topic and queueId are stored in the properties of the message. Because the replace of the topic, the message will not be forwarded to the Consumer Queue of the original topic, and the consumer cannot perceive the existence of the message and will not consume it. In fact, changing the topic is the conventional method of RocketMQ(just recall the implementation mechanism of the delay message).
 
-2. Commit/Rollback operation and introduction of Op message  
+2 Commit/Rollback operation and introduction of Op message  
   
   After finishing writing a message that is invisible to the user in the first phase, here comes two cases in the second phase. One is Commit operation, after which the message needs to be visible to the user; the other one is Rollback operation, after which the first phase message(Half message) needs to be revoked. For the case of Rollback, since first-phase message itself is invisible to the user, there is no need to actually revoke the message (in fact, RocketMQ can't actually delete a message because it is a sequential-write file). But still some operation needs to be done to identity the final status of the message, to differ it from pending status message. To do this, the concept of "Op message" is introduced, which means the message has a certain status(Commit or Rollback). If a transaction message does not have a corresponding Op message, the status of the transaction is still undetermined (probably the second-phase failed). By introducing the Op message, the RocketMQ records an Op message for every Half message regardless it is Commit or Rollback. The only difference between Commit and Rollback is that when it comes to Commit, the index of the Half message is created before the Op message is written.
 
-3. How Op message stored and the correspondence between Op message and Half message  
+3 How Op message stored and the correspondence between Op message and Half message  
   
-  RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation.
+  RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionalMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation.
 
 ![](../cn/image/rocketmq_design_12.png)
 
-4. Index construction of Half messages  
+4 Index construction of Half messages  
   
   When performing Commit operation of the second phase, the index of the Half message needs to be built. Since the Half message is written to a special topic(RMQ_SYS_TRANS_HALF_TOPIC) in the first phase of 2PC, so it needs to be read out from the special topic when building index, and replace the topic and queueId with the real target topic and queueId, and then write through a normal message that is visible to the user. Therefore, in conclusion, the second phase recovers a complete normal message using the content of the Half message stored in the first phase, and then goes through the message-writing process.
 
-5. How to handle the message failed in the second phase？
+5 How to handle the message failed in the second phase？
   
   If commit/rollback phase fails, for example, a network problem causes the Commit to fail when you do Commit. Then certain strategy is required to make sure the message finally commit. RocketMQ uses a compensation mechanism called "back-check". The broker initiates a back-check request for the message in pending status, and sends the request to the corresponding producer side (the same producer group as the producer group who sent the Half message). The producer checks the status of local transaction and redo Commit or Rollback. The broker performs the back-check by comparing the RMQ_SYS_TRANS_HALF_TOPIC messages and the RMQ_SYS_TRANS_OP_HALF_TOPIC messages and advances the checkpoint(recording those transaction messages that the status are certain).
 

docs/en/Example_Delay.md

@@ -1,6 +1,6 @@
 # Schedule example
 
-### 1.Start consumer to wait for incoming subscribed messages 
+### 1 Start consumer to wait for incoming subscribed messages 
 
 ```java
 import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
@@ -35,7 +35,7 @@ public class ScheduledMessageConsumer {
 }
 ```
 
-### 2.Send scheduled messages 
+### 2 Send scheduled messages 
 
 ```java
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
@@ -64,15 +64,15 @@ public class ScheduledMessageProducer {
 }
 ```
 
-### 3.Verification 
+### 3 Verification 
 
 You should see messages are consumed about 10 seconds later than their storing time. 
 
-### 4.Use scenarios for scheduled messages
+### 4 Use scenarios for scheduled messages
 
 For example, in e-commerce, if an order is submitted, a delay message can be sent, and the status of the order can be checked after 1 hour. If the order is still unpaid, the order can be cancelled and the inventory released.
 
-### 5.Restrictions on the use of scheduled messages
+### 5 Restrictions on the use of scheduled messages
 
 ```java 
 // org/apache/rocketmq/store/config/MessageStoreConfig.java
@@ -82,4 +82,4 @@ private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m
 
 Nowadays RocketMq does not support any time delay. It needs to set several fixed delay levels, which correspond to level 1 to 18 from 1s to 2h. Message consumption failure will enter the delay message queue. Message sending time is related to the set delay level and the number of retries.
 
- See `SendMessageProcessor.java` 
+ See `SendMessageProcessor.java` 

docs/en/Example_Filter.md

@@ -1,7 +1,7 @@
 # Filter Example
 ----------
 
-In most cases, tag is a simple and useful design to select message you want. For example:
+In most cases, tag is a simple and useful design to select messages you want. For example:
 
 ```java
 DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_EXAMPLE");
@@ -28,7 +28,7 @@ SQL feature could do some calculation through the properties you put in when sen
 ------------
 ```
 
-## 1.Grammars
+## 1 Grammars
 RocketMQ only defines some basic grammars to support this feature. You could also extend it easily.
 
 - Numeric comparison, like **>**, **>=**, **<**, **<=**, **BETWEEN**, **=**;
@@ -43,13 +43,13 @@ Constant types are:
 - **NULL**, special constant;
 - Boolean, **TRUE** or **FALSE**;
 
-## 2.Usage constraints
+## 2 Usage constraints
 Only push consumer could select messages by SQL92. The interface is:
 ```
 public void subscribe(finalString topic, final MessageSelector messageSelector)
 ```
 
-## 3.Producer example
+## 3 Producer example
 You can put properties in message through method putUserProperty when sending.
 
 ```java
@@ -67,7 +67,7 @@ producer.shutdown();
 
 ```
 
-## 4.Consumer example
+## 4 Consumer example
 Use `MessageSelector.bySql` to select messages through SQL when consuming.
 
 

docs/en/Example_Orderly.md

@@ -1,4 +1,4 @@
-# Example for ordered messages
+# Example for Ordered Messages
 
 RocketMQ provides ordered messages using FIFO order. All related messages need to be sent into the same message queue in an orderly manner.
 

docs/en/Example_Transaction.md

@@ -1,4 +1,4 @@
-# Transaction message example 
+# Transaction Message Example 
 
 ## 1 Transaction message status 
 There are three states for transaction message:  

docs/en/FAQ.md

@@ -2,34 +2,39 @@
 
 The following questions are frequently asked with regard to the RocketMQ project in general.
 
-## General
+## 1 General
 
-### 1. Why did we create rocketmq project instead of selecting other products?
+1. Why did we create rocketmq project instead of selecting other products?
 
-Please refer to [Why RocketMQ](http://rocketmq.apache.org/docs/motivation)
+   Please refer to [Why RocketMQ](http://rocketmq.apache.org/docs/motivation)
 
-### 2. Do I have to install other softeware, such as zookeeper, to use RocketMQ?
+2. Do I have to install other softeware, such as zookeeper, to use RocketMQ?
 
-No. RocketMQ can run independently.
+   No. RocketMQ can run independently.
 
-## Usage
+## 2 Usage
 
 ### 1. Where does the newly created Consumer ID start consuming messages?
 
-1. If the topic sends a message within three days, then the consumer start consuming messages from the first message saved in the server.
-2. If the topic sends a message three days ago, the consumer starts to consume messages from the latest message in the server, in other words, starting from the tail of message queue.
-3. If such consumer is rebooted, then it starts to consume messages from the last consumption location.
+ 1) If the topic sends a message within three days, then the consumer start consuming messages from the first message saved in the server.
+  
+ 2) If the topic sends a message three days ago, the consumer starts to consume messages from the latest message in the server, in other words, starting from the tail of message queue.
+  
+ 3) If such consumer is rebooted, then it starts to consume messages from the last consumption location.
 
 ### 2. How to reconsume message when consumption fails?
 
-1. Cluster consumption pattern, The consumer business logic code returns Action.ReconsumerLater, NULL, or throws an exception, if a message failed to be consumed, it will retry for up to 16 times, after that, the message would be descarded.
-2. Broadcast consumption patternThe broadcaset consumption still ensures that a message is consumered at least once, but no resend option is provided.
+ 1) Cluster consumption pattern, The consumer business logic code returns Action.ReconsumerLater, NULL, or throws an exception, if a message failed to be consumed, it will retry for up to 16 times, after that, the message would be descarded.
+  
+ 2) Broadcast consumption patternThe broadcaset consumption still ensures that a message is consumered at least once, but no resend option is provided.
 
 ### 3. How to query the failed message if there is a consumption failure?
 
-1. Using topic query by time, you can query messages within a period of time.
-2. Using Topic and Message Id to accurately query the message.
-3. Using Topic and Message Key accurately query a class of messages with the same Message Key.
+ 1) Using topic query by time, you can query messages within a period of time.
+  
+ 2) Using Topic and Message Id to accurately query the message.
+  
+ 3) Using Topic and Message Key accurately query a class of messages with the same Message Key.
 
 ### 4. Are messages delivered exactly once?
 
@@ -37,10 +42,11 @@ RocketMQ ensures that all messages are delivered at least once. In most cases, t
 
 ### 5. How to add a new broker?
 
-1. Start up a new broker and register it to the same list of name servers.
-2. By default, only internal system topics and consumer groups are created automatically. If you would like to have your business topic and consumer groups on the new node, please replicate them from the existing broker. Admin tool and command lines are provided to handle this.
+ 1) Start up a new broker and register it to the same list of name servers.
+  
+ 2) By default, only internal system topics and consumer groups are created automatically. If you would like to have your business topic and consumer groups on the new node, please replicate them from the existing broker. Admin tool and command lines are provided to handle this.
 
-## Configuration related
+## 3 Configuration related
 
 The following answers are all default values and can be modified by configuration.
 
@@ -60,7 +66,7 @@ consumer.setConsumeThreadMin(20);
 consumer.setConsumeThreadMax(20);
 ```
 
-## Errors
+## 4 Errors
 
 ### 1. If you start a producer or consumer failed and the error message is producer group or consumer repeat.
 
@@ -76,15 +82,15 @@ Solution: Fastjson version has to be upgraded to rocketmq client dependent versi
 
 ### 3. What is the impact of a broker crash.
 
-1. Master crashes
+ 1) Master crashes
 
 Messages can no longer be sent to this broker set, but if you have another broker set available, messages can still be sent given the topic is present. Messages can still be consumed from slaves.
 
-2. Some slave crash
+ 2) Some slave crash
 
 As long as there is another working slave, there will be no impact on sending messages. There will also be no impact on consuming messages except when the consumer group is set to consume from this slave preferably. By default, comsumer group consumes from master.
 
-3. All slaves crash
+ 3) All slaves crash
 
 There will be no impact on sending messages to master, but, if the master is SYNC_MASTER, producer will get a SLAVE_NOT_AVAILABLE indicating that the message is not sent to any slaves. There will also be no impact on consuming messages except that if the consumer group is set to consume from slave preferably. By default, comsumer group consumes from master.
 
@@ -92,9 +98,12 @@ There will be no impact on sending messages to master, but, if the master is SYN
 
 This happens when you are trying to send messages to a topic whose routing info is not available to the producer.
 
-1. Make sure that the producer can connect to a name server and is capable of fetching routing meta info from it.
-2. Make sure that name servers do contain routing meta info of the topic. You may query the routing meta info from name server through topicRoute using admin tools or web console.
-3. Make sure that your brokers are sending heartbeats to the same list of name servers your producer is connecting to.
-4. Make sure that the topic’s permssion is 6(rw-), or at least 2(-w-).
+ 1) Make sure that the producer can connect to a name server and is capable of fetching routing meta info from it.
+  
+ 2) Make sure that name servers do contain routing meta info of the topic. You may query the routing meta info from name server through topicRoute using admin tools or web console.
+  
+ 3) Make sure that your brokers are sending heartbeats to the same list of name servers your producer is connecting to.
+  
+ 4) Make sure that the topic’s permssion is 6(rw-), or at least 2(-w-).
 
 If you can’t find this topic, create it on a broker via admin tools command updateTopic or web console.

docs/en/Feature.md

@@ -1,8 +1,8 @@
 # Features
-## Subscribe and Publish
+## 1 Subscribe and Publish
 Message publication refers to that a producer sends messages to a topic; Message subscription means a consumer follows a topic with certain tags and then consumes data from that topic.
 
-## Message Ordering
+## 2 Message Ordering
 Message ordering refers to that a group of messages can be consumed orderly as they are published. For example, an order generates three messages: order creation, order payment, and order completion. It only makes sense to consume them in their generated order, but orders can be consumed in parallel at the same time. RocketMQ can strictly guarantee these messages are in order.
 
 Orderly message are divided into global orderly message and partitioned orderly message. Global order means that all messages under a certain topic must be in order, partitioned order only requires each group of messages are consumed orderly.
@@ -13,32 +13,32 @@ Applicable scenario: the performance requirement is not high, and all messages a
 For a given Topic, all messages are partitioned according to sharding key. Messages within the same partition are published and consumed in strict FIFO order. Sharding key is the key field to distinguish message's partition, which is a completely different concept from the key of ordinary messages.
 Applicable scenario: high performance requirement, with sharding key as the partition field, messages within the same partition are published and consumed according to FIFO principle strictly.
 
-## Message Filter
+## 3 Message Filter
 Consumers of RocketMQ can filter messages based on tags as well as support for user-defined attribute filtering. Message filter is currently implemented on the Broker side, with the advantage of reducing the network transmission of useless messages for Consumer and the disadvantage of increasing the burden on the Broker and relatively complex implementation.
 
-## Message Reliability
+## 4 Message Reliability
 RocketMQ supports high reliability of messages in several situations:
-1) Broker shutdown normally
-2) Broker abnormal crash
-3) OS Crash
-4) The machine is out of power, but it can be recovered immediately
-5) The machine cannot be started up (the CPU, motherboard, memory and other key equipment may be damaged)
-6) Disk equipment damaged
+1 Broker shutdown normally
+2 Broker abnormal crash
+3 OS Crash
+4 The machine is out of power, but it can be recovered immediately
+5 The machine cannot be started up (the CPU, motherboard, memory and other key equipment may be damaged)
+6 Disk equipment damaged
 
 In the four cases of 1), 2), 3), and 4) where the hardware resource can be recovered immediately, RocketMQ guarantees that the message will not be lost or a small amount of data will be lost (depending on whether the flush disk type is synchronous or asynchronous)
 
 5) and 6) are single point of failure and cannot be recovered. Once it happens, all messages on the single point will be lost. In both cases, RocketMQ ensures that 99% of the messages are not lost through asynchronous replication, but a very few number of messages may still be lost. Synchronous double write mode can completely avoid single point of failure, which will surely affect the performance and suitable for the occasion of high demand for message reliability, such as money related applications. Note: RocketMQ supports synchronous double writes since version 3.0.
 
-## At Least Once
+## 5 At Least Once
 At least Once refers to that every message will be delivered at least once. RocketMQ supports this feature because the Consumer pulls the message locally and does not send an ack back to the server until it has consumed it.
 
-## Backtracking Consumption
+## 6 Backtracking Consumption
 Backtracking consumption refers to that the Consumer has consumed the message successfully, but the business needs to consume again. To support this function, the message still needs to be retained after the Broker sends the message to the Consumer successfully. The re-consumption is normally based on time dimension. For example, after the recovery of the Consumer system failure, the data one hour ago needs to be re-consumed, then the Broker needs to provide a mechanism to reverse the consumption progress according to the time dimension. RocketMQ supports backtracking consumption by time trace, with the time dimension down to milliseconds.
 
-## Transactional Message
+## 7 Transactional Message
 RocketMQ transactional message refers to the fact that the application of a local transaction and the sending of a Message operation can be defined in a global transaction which means both succeed or fail simultaneously. RocketMQ transactional message provides distributed transaction functionality similar to X/Open XA, enabling the ultimate consistency of distributed transactions through transactional message.
 
-## Scheduled Message
+## 8 Scheduled Message
 Scheduled message(delay queue) refers to that messages are not consumed immediately after they are sent to the broker, but waiting to be delivered to the real topic after a specific time.
 The broker has a configuration item, messageDelayLevel, with default values “1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h”, 18 levels. Users can configure a custom messageDelayLevel. Note that messageDelayLevel is a broker's property rather than a topic's. When sending a message, just set the delayLevel level: msg.setDelayLevel(level). There are three types of levels:
 
@@ -50,21 +50,21 @@ Scheduled messages are temporarily saved in a topic named SCHEDULE_TOPIC_XXXX, a
 
 Note that Scheduled messages are counted both the first time they are written and the time they are scheduled to be written to the real topic, so both the send number and the TPS are increased.
 
-## Message Retry
+## 9 Message Retry
 When the Consumer fails to consume the message, a retry mechanism is needed to make the message to be consumed again. Consumer's consume failure can usually be classified as follows:
 - due to the reasons of the message itself, such as deserialization failure, the message data itself cannot be processed (for example, the phone number of the current message is cancelled and cannot be charged), etc. This kind of error usually requires skipping this message and consuming others since immediately retry would be failed 99%, so it is better to provide a timed retry mechanism that retries after 10 seconds.
 - due to the reasons of dependent downstream application services are not available, such as db connection is not usable, perimeter network is not unreachable, etc. When this kind of error is encountered, consuming other messages will also result in an error even if the current failed message is skipped. In this case, it is recommended to sleep for 30s before consuming the next message, which will reduce the pressure on the broker to retry the message.
 
 RocketMQ will set up a retry queue named “%RETRY%+consumerGroup” for each consumer group(Note that the retry queue for this topic is for consumer groups, not for each topic) to temporarily save messages cannot be consumed by customer due to all kinds of reasons. Considering that it takes some time for the exception to recover, multiple retry levels are set for the retry queue, and each retry level has a corresponding re-deliver delay. The more retries, the greater the deliver delay. RocketMQ first save retry messages to the delay queue which topic is named “SCHEDULE_TOPIC_XXXX”, then background schedule task will save the messages to “%RETRY%+consumerGroup” retry queue according to their corresponding delay.   
 
-## Message Resend
+## 10 Message Resend
 When a producer sends a message, the synchronous message will be resent if fails, the asynchronous message will retry and oneway message is without any guarantee. Message resend ensures that messages are sent successfully and without lost as much as possible, but it can lead to message duplication, which is an unavoidable problem in RocketMQ. Under normal circumstances, message duplication will not occur, but when there is a large number of messages and network jitter, message duplication will be a high-probability event. In addition, producer initiative messages resend and the consumer load changes will also result in duplicate messages. The message retry policy can be set as follows:
 
 - retryTimesWhenSendFailed: Synchronous message retry times when send failed, default value is 2, so the producer will try to send retryTimesWhenSendFailed + 1 times at most. To ensure that the message is not lost, producer will try sending the message to another broker instead of selecting the broker that failed last time. An exception will be thrown if it reaches the retry limit, and the client should guarantee that the message will not be lost. Messages will resend when RemotingException, MQClientException, and partial MQBrokerException occur.
 - retryTimesWhenSendAsyncFailed: Asynchronous message retry times when send failed, asynchronous retry sends message to the same broker instead of selecting another one and does not guarantee that the message wont lost. 
 - retryAnotherBrokerWhenNotStoreOK: Message flush disk (master or slave) timeout or slave not available (return status is not SEND_OK), whether to try to send to another broker, default value is false. Very important messages can set to true.
 
-## Flow Control
+## 11 Flow Control
 Producer flow control, because broker processing capacity reaches a bottleneck; Consumer flow control, because the consumption capacity reaches a bottleneck.
 
 Producer flow control:
@@ -80,7 +80,7 @@ Consumer flow control:
 
 The result of consumer flow control is to reduce the pull frequency.
 
-## Dead Letter Queue
+## 12 Dead Letter Queue
 Dead letter queue is used to deal messages that cannot be consumed normally. When a message is consumed failed at first time, the message queue will automatically resend the message. If the consumption still fails after the maximum number retry, it indicates that the consumer cannot properly consume the message under normal circumstances. At this time, the message queue will not immediately abandon the message, but send it to the special queue corresponding to the consumer.
 
 RocketMQ defines the messages that could not be consumed under normal circumstances as Dead-Letter Messages, and the special queue in which the Dead-Letter Messages are saved as Dead-Letter Queues. In RocketMQ, the consumer instance can consume again by resending messages in the Dead-Letter Queue using console.

docs/en/Operations_Consumer.md

@@ -2,24 +2,24 @@
 
 ----
 
-### 2.1 Consumption process idempotent
+### 1 Consumption process idempotent
 
 RocketMQ cannot avoid Exactly-Once, so if the business is very sensitive to consumption repetition, it is important to perform deduplication at the business level. Deduplication can be done with a relational database. First, you need to determine the unique key of the message, which can be either msgId or a unique identifier field in the message content, such as the order Id. Determine if a unique key exists in the relational database before consumption. If it does not exist, insert it and consume it, otherwise skip it. (The actual process should consider the atomic problem, determine whether there is an attempt to insert, if the primary key conflicts, the insertion fails, skip directly)
 
-### 2.2  Slow message processing
+### 2  Slow message processing
 
-#### 2.2.1 Increase consumption parallelism
+#### 2.1 Increase consumption parallelism
 
 Most messages consumption behaviors are IO-intensive, That is, it may be to operate the database, or call RPC. The consumption speed of such consumption behavior lies in the throughput of the back-end database or the external system. By increasing the consumption parallelism, the total consumption throughput can be increased, but the degree of parallelism is increased to a certain extent. Instead it will fall.Therefore, the application must set a reasonable degree of parallelism. There are several ways to modify the degree of parallelism of consumption as follows:
 
 * Under the same ConsumerGroup, increase the degree of parallelism by increasing the number of Consumer instances (note that the Consumer instance that exceeds the number of subscription queues is invalid). Can be done by adding machines, or by starting multiple processes on an existing machine.
 * Improve the consumption parallel thread of a single Consumer by modifying the parameters consumeThreadMin and consumeThreadMax.
 
-#### 2.2.2 Batch mode consumption
+#### 2.2 Batch mode consumption
 
 Some business processes can increase consumption throughput to a large extent if they support batch mode consumption. For example, order deduction application, it takes 1s to process one order at a time, and it takes only 2s to process 10 orders at a time. In this way, the throughput of consumption can be greatly improved. By setting the consumer's consumeMessageBatchMaxSize to return a parameter, the default is 1, that is, only one message is consumed at a time, for example, set to N, then the number of messages consumed each time is less than or equal to N.
 
-#### 2.2.3 Skip non-critical messages
+#### 2.3 Skip non-critical messages
 
 When a message is accumulated, if the consumption speed cannot keep up with the transmission speed, if the service does not require high data, you can choose to discard the unimportant message. For example, when the number of messages in a queue is more than 100,000 , try to discard some or all of the messages, so that you can quickly catch up with the speed of sending messages. The sample code is as follows:
 
@@ -40,7 +40,7 @@ public ConsumeConcurrentlyStatus consumeMessage(
 }
 ```
 
-#### 2.2.4 Optimize each message consumption process
+#### 2.4 Optimize each message consumption process
 
 For example, the consumption process of a message is as follows:
 
@@ -52,7 +52,7 @@ For example, the consumption process of a message is as follows:
 
 There are 4 interactions with the DB in the consumption process of this message. If it is calculated by 5ms each time, it takes a total of 20ms. If the business calculation takes 5ms, then the total time is 25ms, So if you can optimize 4 DB interactions to 2 times, the total time can be optimized to 15ms, which means the overall performance is increased by 40%. Therefore, if the application is sensitive to delay, the DB can be deployed on the SSD hard disk. Compared with the SCSI disk, the former RT will be much smaller.
 
-### 2.3 Print Log
+### 3 Print Log
 
 If the amount of messages is small, it is recommended to print the message in the consumption entry method, consume time, etc., to facilitate subsequent troubleshooting.
 
@@ -68,33 +68,33 @@ public ConsumeConcurrentlyStatus consumeMessage(
 
 If you can print the time spent on each message, it will be more convenient when troubleshooting online problems such as slow consumption.
 
-### 2.4 Other consumption suggestions
+### 4 Other consumption suggestions
 
-#### 2.4.1、Consumer Group and Subscriptions
+#### 4.1、Consumer Group and Subscriptions
 
 The first thing you should be aware of is that different Consumer Group can consume the same topic independently, and each of them will have their own consuming offsets. Please make sure each Consumer within the same Group to subscribe the same topics.
 
-#### 2.4.2、Orderly
+#### 4.2、Orderly
 
 The Consumer will lock each MessageQueue to make sure it is consumed one by one in order. This will cause a performance loss, but it is useful when you care about the order of the messages. It is not recommended to throw exceptions, you can return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT instead.
 
-#### 2.4.3、Concurrently
+#### 4.3、Concurrently
 
 As the name tells, the Consumer will consume the messages concurrently. It is recommended to use this for good performance. It is not recommended to throw exceptions, you can return ConsumeConcurrentlyStatus.RECONSUME_LATER instead.
 
-#### 2.4.4、Consume Status
+#### 4.4、Consume Status
 
 For MessageListenerConcurrently, you can return RECONSUME_LATER to tell the consumer that you can not consume it right now and want to reconsume it later. Then you can continue to consume other messages. For MessageListenerOrderly, because you care about the order, you can not jump over the message, but you can return SUSPEND_CURRENT_QUEUE_A_MOMENT to tell the consumer to wait for a moment.
 
-#### 2.4.5、Blocking
+#### 4.5、Blocking
 
 It is not recommend to block the Listener, because it will block the thread pool, and eventually may stop the consuming process.
 
-#### 2.4.6、Thread Number
+#### 4.6、Thread Number
 
 The consumer use a ThreadPoolExecutor to process consuming internally, so you can change it by setting setConsumeThreadMin or setConsumeThreadMax.
 
-#### 2.4.7、ConsumeFromWhere
+#### 4.7、ConsumeFromWhere
 
 When a new Consumer Group is established, it will need to decide whether it needs to consume the historical messages which had already existed in the Broker. CONSUME_FROM_LAST_OFFSET will ignore the historical messages, and consume anything produced after that. CONSUME_FROM_FIRST_OFFSET will consume every message existed in the Broker. You can also use CONSUME_FROM_TIMESTAMP to consume messages produced after the specified timestamp.
 

docs/en/Operations_Producer.md

@@ -2,7 +2,7 @@
 ----
 ##### 1 Message Sending Tips
 ###### 1.1 The Use of Tags
-One application instance should use one topic as much as possible and the subtype of messages can be marked by tags. Tag provides extra flexibility to users. In the consume subscribing process, the messages filtering can only be handled by using tags when the tags are specified in the message sending process: `message.setTags("TagA")`
+One application instance should use one topic as much as possible and the subtype of messages can be marked by tags. Tag provides extra flexibility to users. In the consume subscribing process, the messages filtering can only be handled by using tags when the tags are specified in the message sending process: `message.setTags("TagA")`.
 ###### 1.2 The Use of Keys
 A business  key can be set in one message and it will be easier to look up the message on a broker server to diagnose issues during development. Each message will be created index(hash index) by server, instance can query the content of this message by topic and key and who consumes the message.Because of the hash index, make sure that the key should be unique in order to avoid potential hash index conflict.
 ``` java
@@ -41,4 +41,4 @@ The message sending is usually a process like below:
 * Sever handles request
 * Sever returns response to client
 
-The total costing time of sending one message is the sum of costing time of three steps above. Some situations demand that total costing time must be in a quite low level, however, do not take reliable performance into consideration, such as log collection. This kind of application could be called in one-way mode, which means client sends request but not wait for response. In this kind of mode, the cost of sending request is only a call of system operation which means one operation writing data to client socket buffer. Generally, the time cost of this process will be controlled n microseconds level.
+The total costing time of sending one message is the sum of costing time of three steps above. Some situations demand that total costing time must be in a quite low level, however, do not take reliable performance into consideration, such as log collection. This kind of application could be called in one-way mode, which means client sends request but not wait for response. In this kind of mode, the cost of sending request is only a call of system operation which means one operation writing data to client socket buffer. Generally, the time cost of this process will be controlled n microseconds level.

docs/en/Operations_Trace.md

@@ -1,10 +1,6 @@
 # Message Trace
 
-## Introduction
-
-This document focuses on how to quickly deploy and use RocketMQ clusters that support message trace features
-
-## 1. Key Attributes of Message Trace Data
+## 1 Key Attributes of Message Trace Data
 
 | Producer        | Consumer        | Broker     |
 | ---------------- | ----------------- | ------------ |
@@ -13,7 +9,7 @@ This document focuses on how to quickly deploy and use RocketMQ clusters that su
 | whether the message was sent successfully | Whether the message was successfully consumed  | The Key of the message  |
 | Time spent sending         | Time spent consuming         | Tag of the message  |
 
-## 2. Support for Message Trace Cluster Deployment
+## 2 Support for Message Trace Cluster Deployment
 
 ### 2.1 Broker Configuration Fille
 
@@ -46,7 +42,7 @@ For scenarios with large amount of trace message data , one of the Broker nodes
 ### 2.4 Start the Broker that Starts the MessageTrace
 `nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &`
 
-## 3. Save the Topic Definition of Message Trace 
+## 3 Save the Topic Definition of Message Trace 
 RocketMQ's message trace feature supports two ways to store trace data:
 
 ### 3.1 System-level TraceTopic
@@ -55,7 +51,7 @@ By default, message track data is stored in the system-level TraceTopic(names：
 ### 3.2 Custom TraceTopic 
 If the user is not prepared to store the message track data in the system-level default TraceTopic, you can also define and create a user-level Topic to save the track (that is, to create a regular Topic to save the message track data)。The following section introduces how the Client interface supports the user-defined TraceTopic.
 
-## 4. Client Practices that Support Message Trace
+## 4 Client Practices that Support Message Trace
 In order to reduce as much as possible the transformation work of RocketMQ message trace feature used in the user service system, the author added a switch parameter (**enableMsgTrace**) to the original interface in the design to realize whether the message trace is opened or not.
 
 ### 4.1 Opening  the Message Trace when Sending  the Message

docs/en/Troubleshoopting.md

@@ -1,6 +1,6 @@
 # Operation FAQ
 
-## 1. RocketMQ's mqadmin command error.
+## 1 RocketMQ's mqadmin command error.
 
 >  Problem: Sometimes after deploying the RocketMQ cluster, when you try to execute some commands of "mqadmin", the following exception will appear：
 >
@@ -10,7 +10,7 @@
 
 Solution: Execute  `export NAMESRV_ADDR=ip:9876` (ip refers to the address of NameServer deployed in the cluster) on the VM that deploys the RocketMQ cluster.Then you will execute commands of "mqadmin" successfully.
 
-## 2. The inconsistent version of RocketMQ between the producer and consumer leads to the problem that message can't be consumed normally.
+## 2 The inconsistent version of RocketMQ between the producer and consumer leads to the problem that message can't be consumed normally.
 
 > Problem: The same producer sends a message, consumer A can consume, but consumer B can't consume, and the RocketMQ Console appears:
 >
@@ -20,7 +20,7 @@ Solution: Execute  `export NAMESRV_ADDR=ip:9876` (ip refers to the address of Na
 
 Solution: The jar package of RocketMQ, such as rocketmq-client, should be the same version on the consumer and producer.
 
-## 3. When adding a new topic consumer group, historical messages can't be consumed.
+## 3 When adding a new topic consumer group, historical messages can't be consumed.
 
 > Problem: When a new consumer group of the same topic is started, the consumed message is the current offset message, and the historical message is not obtained.
 
@@ -50,11 +50,11 @@ consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
 consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP);
 ```
 
-## 4. How to enable reading data from Slave
+## 4 How to enable reading data from Slave
 
 In some cases, the Consumer needs to reset the consume position to 1-2 days ago. At this time, on the Master Broker with limited memory, the CommitLog will carry a relatively heavy IO pressure, affecting the reading and writing of other messages on that Broker. You can enable `slaveReadEnable=true`. When Master Broker finds that the difference between the Consumer's consume position and the latest value of CommitLog exceeds the percentage of machine's memory (`accessMessageInMemoryMaxRatio=40%`), it will recommend Consumer  to read from Slave Broker and relieve Master Broker's IO.
 
-## 5. Performance
+## 5 Performance
 
 Asynchronous flush disk is recommended to use spin lock.
 
@@ -64,7 +64,7 @@ Asynchronous flush disk is recommended to open `TransientStorePoolEnable` and cl
 
 Synchronous flush disk is recommended to increase the `sendMessageThreadPoolNums` appropriately. The specific configuration needs to be tested.
 
-## 6. The meaning and difference between msgId and offsetMsgId in RocketMQ
+## 6 The meaning and difference between msgId and offsetMsgId in RocketMQ
 
 After sending message with RocketMQ, you will usually see the following log:
 

namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java

@@ -128,6 +128,17 @@ public class RouteInfoManager {
                     brokerData = new BrokerData(clusterName, brokerName, new HashMap<Long, String>());
                     this.brokerAddrTable.put(brokerName, brokerData);
                 }
+                Map<Long, String> brokerAddrsMap = brokerData.getBrokerAddrs();
+                //Switch slave to master: first remove <1, IP:PORT> in namesrv, then add <0, IP:PORT>
+                //The same IP:PORT must only have one record in brokerAddrTable
+                Iterator<Entry<Long, String>> it = brokerAddrsMap.entrySet().iterator();
+                while (it.hasNext()) {
+                    Entry<Long, String> item = it.next();
+                    if (null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey()) {
+                        it.remove();
+                    }
+                }
+
                 String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr);
                 registerFirst = registerFirst || (null == oldAddr);
 

pom.xml

@@ -100,8 +100,8 @@
         <maven.test.skip>false</maven.test.skip>
         <maven.javadoc.skip>true</maven.javadoc.skip>
         <!-- Compiler settings properties -->
-        <maven.compiler.source>1.7</maven.compiler.source>
-        <maven.compiler.target>1.7</maven.compiler.target>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
         <sonar.java.coveragePlugin>jacoco</sonar.java.coveragePlugin>
         <!-- Exclude all generated code -->
         <sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>
@@ -259,6 +259,7 @@
                         <exclude>*/target/**</exclude>
                         <exclude>*/*.iml</exclude>
                         <exclude>docs/**</exclude>
+                        <exclude>localbin/**</exclude>
                     </excludes>
                 </configuration>
             </plugin>

store/pom.xml

@@ -28,6 +28,21 @@
     <name>rocketmq-store ${project.version}</name>
 
     <dependencies>
+        <dependency>
+            <groupId>io.openmessaging.storage</groupId>
+            <artifactId>dledger</artifactId>
+            <version>0.1</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.rocketmq</groupId>
+                    <artifactId>rocketmq-remoting</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>rocketmq-common</artifactId>

store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java

@@ -120,16 +120,9 @@ public class AllocateMappedFileService extends ServiceThread {
         return AllocateMappedFileService.class.getSimpleName();
     }
 
+    @Override
     public void shutdown() {
-        this.stopped = true;
-        this.thread.interrupt();
-
-        try {
-            this.thread.join(this.getJointime());
-        } catch (InterruptedException e) {
-            log.error("Interrupted", e);
-        }
-
+        super.shutdown(true);
         for (AllocateRequest req : this.requestTable.values()) {
             if (req.mappedFile != null) {
                 log.info("delete pre allocated maped file, {}", req.mappedFile.getFileName());

store/src/main/java/org/apache/rocketmq/store/CommitLog.java

@@ -45,11 +45,11 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService;
 public class CommitLog {
     // Message's MAGIC CODE daa320a7
     public final static int MESSAGE_MAGIC_CODE = -626843481;
-    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
+    protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
     // End of file empty MAGIC CODE cbd43194
-    private final static int BLANK_MAGIC_CODE = -875286124;
-    private final MappedFileQueue mappedFileQueue;
-    private final DefaultMessageStore defaultMessageStore;
+    protected final static int BLANK_MAGIC_CODE = -875286124;
+    protected final MappedFileQueue mappedFileQueue;
+    protected final DefaultMessageStore defaultMessageStore;
     private final FlushCommitLogService flushCommitLogService;
 
     //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods
@@ -57,11 +57,11 @@ public class CommitLog {
 
     private final AppendMessageCallback appendMessageCallback;
     private final ThreadLocal<MessageExtBatchEncoder> batchEncoderThreadLocal;
-    private HashMap<String/* topic-queueid */, Long/* offset */> topicQueueTable = new HashMap<String, Long>(1024);
-    private volatile long confirmOffset = -1L;
+    protected HashMap<String/* topic-queueid */, Long/* offset */> topicQueueTable = new HashMap<String, Long>(1024);
+    protected volatile long confirmOffset = -1L;
 
     private volatile long beginTimeInLock = 0;
-    private final PutMessageLock putMessageLock;
+    protected final PutMessageLock putMessageLock;
 
     public CommitLog(final DefaultMessageStore defaultMessageStore) {
         this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(),
@@ -212,6 +212,12 @@ public class CommitLog {
                 log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset);
                 this.defaultMessageStore.truncateDirtyLogicFiles(processOffset);
             }
+        } else {
+            // Commitlog case files are deleted
+            log.warn("The commitlog files are deleted, and delete the consume queue files");
+            this.mappedFileQueue.setFlushedWhere(0);
+            this.mappedFileQueue.setCommittedWhere(0);
+            this.defaultMessageStore.destroyLogics();
         }
     }
 
@@ -366,7 +372,7 @@ public class CommitLog {
         return new DispatchRequest(-1, false /* success */);
     }
 
-    private static int calMsgLength(int bodyLength, int topicLength, int propertiesLength) {
+    protected static int calMsgLength(int bodyLength, int topicLength, int propertiesLength) {
         final int msgLen = 4 //TOTALSIZE
             + 4 //MAGICCODE
             + 4 //BODYCRC
@@ -396,6 +402,7 @@ public class CommitLog {
         this.confirmOffset = phyOffset;
     }
 
+    @Deprecated
     public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) {
         // recover by the minimum time stamp
         boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover();
@@ -456,7 +463,7 @@ public class CommitLog {
                         }
                     }
                 } else {
-                    log.info("recover physics file end, " + mappedFile.getFileName());
+                    log.info("recover physics file end, " + mappedFile.getFileName() + " pos=" + byteBuffer.position());
                     break;
                 }
             }
@@ -474,6 +481,7 @@ public class CommitLog {
         }
         // Commitlog case files are deleted
         else {
+            log.warn("The commitlog files are deleted, and delete the consume queue files");
             this.mappedFileQueue.setFlushedWhere(0);
             this.mappedFileQueue.setCommittedWhere(0);
             this.defaultMessageStore.destroyLogics();

store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java

@@ -106,7 +106,7 @@ public class ConsumeQueue {
 
                     if (offset >= 0 && size > 0) {
                         mappedFileOffset = i + CQ_STORE_UNIT_SIZE;
-                        this.maxPhysicOffset = offset;
+                        this.maxPhysicOffset = offset + size;
                         if (isExtAddr(tagsCode)) {
                             maxExtAddr = tagsCode;
                         }
@@ -224,7 +224,7 @@ public class ConsumeQueue {
 
         int logicFileSize = this.mappedFileSize;
 
-        this.maxPhysicOffset = phyOffet - 1;
+        this.maxPhysicOffset = phyOffet;
         long maxExtAddr = 1;
         while (true) {
             MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
@@ -249,7 +249,7 @@ public class ConsumeQueue {
                             mappedFile.setWrotePosition(pos);
                             mappedFile.setCommittedPosition(pos);
                             mappedFile.setFlushedPosition(pos);
-                            this.maxPhysicOffset = offset;
+                            this.maxPhysicOffset = offset + size;
                             // This maybe not take effect, when not every consume queue has extend file.
                             if (isExtAddr(tagsCode)) {
                                 maxExtAddr = tagsCode;
@@ -267,7 +267,7 @@ public class ConsumeQueue {
                             mappedFile.setWrotePosition(pos);
                             mappedFile.setCommittedPosition(pos);
                             mappedFile.setFlushedPosition(pos);
-                            this.maxPhysicOffset = offset;
+                            this.maxPhysicOffset = offset + size;
                             if (isExtAddr(tagsCode)) {
                                 maxExtAddr = tagsCode;
                             }
@@ -348,7 +348,7 @@ public class ConsumeQueue {
                         long tagsCode = result.getByteBuffer().getLong();
 
                         if (offsetPy >= phyMinOffset) {
-                            this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i;
+                            this.minLogicOffset = mappedFile.getFileFromOffset() + i;
                             log.info("Compute logical min offset: {}, topic: {}, queueId: {}",
                                 this.getMinOffsetInQueue(), this.topic, this.queueId);
                             // This maybe not take effect, when not every consume queue has extend file.
@@ -420,7 +420,8 @@ public class ConsumeQueue {
     private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode,
         final long cqOffset) {
 
-        if (offset <= this.maxPhysicOffset) {
+        if (offset + size <= this.maxPhysicOffset) {
+            log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset);
             return true;
         }
 
@@ -464,7 +465,7 @@ public class ConsumeQueue {
                     );
                 }
             }
-            this.maxPhysicOffset = offset;
+            this.maxPhysicOffset = offset + size;
             return mappedFile.appendMessage(this.byteBufferIndex.array());
         }
         return false;

store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java

@@ -42,24 +42,23 @@ import org.apache.rocketmq.common.SystemClock;
 import org.apache.rocketmq.common.ThreadFactoryImpl;
 import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.LoggerName;
-import org.apache.rocketmq.logging.InternalLogger;
-import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
 import org.apache.rocketmq.common.running.RunningStats;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
 import org.apache.rocketmq.store.ha.HAService;
 import org.apache.rocketmq.store.index.IndexService;
 import org.apache.rocketmq.store.index.QueryOffsetResult;
 import org.apache.rocketmq.store.schedule.ScheduleMessageService;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
-import static org.apache.rocketmq.store.config.BrokerRole.SLAVE;
-
 public class DefaultMessageStore implements MessageStore {
     private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
 
@@ -119,7 +118,11 @@ public class DefaultMessageStore implements MessageStore {
         this.messageStoreConfig = messageStoreConfig;
         this.brokerStatsManager = brokerStatsManager;
         this.allocateMappedFileService = new AllocateMappedFileService(this);
-        this.commitLog = new CommitLog(this);
+        if (messageStoreConfig.isEnableDLegerCommitLog()) {
+            this.commitLog = new DLedgerCommitLog(this);
+        } else {
+            this.commitLog = new CommitLog(this);
+        }
         this.consumeQueueTable = new ConcurrentHashMap<>(32);
 
         this.flushConsumeQueueService = new FlushConsumeQueueService();
@@ -127,8 +130,11 @@ public class DefaultMessageStore implements MessageStore {
         this.cleanConsumeQueueService = new CleanConsumeQueueService();
         this.storeStatsService = new StoreStatsService();
         this.indexService = new IndexService(this);
-        this.haService = new HAService(this);
-
+        if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+            this.haService = new HAService(this);
+        } else {
+            this.haService = null;
+        }
         this.reputMessageService = new ReputMessageService();
 
         this.scheduleMessageService = new ScheduleMessageService(this);
@@ -216,29 +222,71 @@ public class DefaultMessageStore implements MessageStore {
 
         lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes()));
         lockFile.getChannel().force(true);
+        {
+            /**
+             * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;
+             * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;
+             * 3. Calculate the reput offset according to the consume queue;
+             * 4. Make sure the fall-behind messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed.
+             */
+            long maxPhysicalPosInLogicQueue = commitLog.getMinOffset();
+            for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
+                for (ConsumeQueue logic : maps.values()) {
+                    if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) {
+                        maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset();
+                    }
+                }
+            }
+            if (maxPhysicalPosInLogicQueue < 0) {
+                maxPhysicalPosInLogicQueue = 0;
+            }
+            if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
+                maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
+                /**
+                 * This happens in following conditions:
+                 * 1. If someone removes all the consumequeue files or the disk get damaged.
+                 * 2. Launch a new broker, and copy the commitlog from other brokers.
+                 *
+                 * All the conditions has the same in common that the maxPhysicalPosInLogicQueue should be 0.
+                 * If the maxPhysicalPosInLogicQueue is gt 0, there maybe something wrong.
+                 */
+                log.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset());
+            }
+            log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}",
+                maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset());
+            this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue);
+            this.reputMessageService.start();
+
+            /**
+             *  1. Finish dispatching the messages fall behind, then to start other services.
+             *  2. DLedger committedPos may be missing, so here just require dispatchBehindBytes <= 0
+             */
+            while (true) {
+                if (dispatchBehindBytes() <= 0) {
+                    break;
+                }
+                Thread.sleep(1000);
+                log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes());
+            }
+            this.recoverTopicQueueTable();
+        }
+
+        if (!messageStoreConfig.isEnableDLegerCommitLog()) {
+            this.haService.start();
+            this.handleScheduleMessageService(messageStoreConfig.getBrokerRole());
+        }
 
         this.flushConsumeQueueService.start();
         this.commitLog.start();
         this.storeStatsService.start();
 
-        if (this.scheduleMessageService != null && SLAVE != messageStoreConfig.getBrokerRole()) {
-            this.scheduleMessageService.start();
-        }
-
-        if (this.getMessageStoreConfig().isDuplicationEnable()) {
-            this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());
-        } else {
-            this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset());
-        }
-        this.reputMessageService.start();
-
-        this.haService.start();
-
         this.createTempFile();
         this.addScheduleTask();
         this.shutdown = false;
     }
 
+
+
     public void shutdown() {
         if (!this.shutdown) {
             this.shutdown = true;
@@ -255,8 +303,9 @@ public class DefaultMessageStore implements MessageStore {
             if (this.scheduleMessageService != null) {
                 this.scheduleMessageService.shutdown();
             }
-
-            this.haService.shutdown();
+            if (this.haService != null) {
+                this.haService.shutdown();
+            }
 
             this.storeStatsService.shutdown();
             this.indexService.shutdown();
@@ -1039,6 +1088,7 @@ public class DefaultMessageStore implements MessageStore {
         return false;
     }
 
+    @Override
     public long dispatchBehindBytes() {
         return this.reputMessageService.behind();
     }
@@ -1320,7 +1370,7 @@ public class DefaultMessageStore implements MessageStore {
         return maxPhysicOffset;
     }
 
-    private void recoverTopicQueueTable() {
+    public void recoverTopicQueueTable() {
         HashMap<String/* topic-queueid */, Long/* offset */> table = new HashMap<String, Long>(1024);
         long minPhyOffset = this.commitLog.getMinOffset();
         for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) {
@@ -1382,6 +1432,18 @@ public class DefaultMessageStore implements MessageStore {
         return brokerStatsManager;
     }
 
+    @Override
+    public void handleScheduleMessageService(final BrokerRole brokerRole) {
+        if (this.scheduleMessageService != null) {
+            if (brokerRole == BrokerRole.SLAVE) {
+                this.scheduleMessageService.shutdown();
+            } else {
+                this.scheduleMessageService.start();
+            }
+        }
+
+    }
+
     public int remainTransientStoreBufferNumbs() {
         return this.transientStorePool.remainBufferNumbs();
     }
@@ -1748,6 +1810,11 @@ public class DefaultMessageStore implements MessageStore {
         }
 
         private void doReput() {
+            if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) {
+                log.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.",
+                    this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset());
+                this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset();
+            }
             for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) {
 
                 if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable()
@@ -1763,7 +1830,7 @@ public class DefaultMessageStore implements MessageStore {
                         for (int readSize = 0; readSize < result.getSize() && doNext; ) {
                             DispatchRequest dispatchRequest =
                                 DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false);
-                            int size = dispatchRequest.getMsgSize();
+                            int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize();
 
                             if (dispatchRequest.isSuccess()) {
                                 if (size > 0) {
@@ -1797,12 +1864,10 @@ public class DefaultMessageStore implements MessageStore {
                                     this.reputFromOffset += size;
                                 } else {
                                     doNext = false;
-                                    if (DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) {
-                                        log.error("[BUG]the master dispatch message to consume queue error, COMMITLOG OFFSET: {}",
-                                            this.reputFromOffset);
+                                    log.error("[BUG]dispatch message to consume queue error, COMMITLOG OFFSET: {}",
+                                        this.reputFromOffset);
 
-                                        this.reputFromOffset += result.getSize() - readSize;
-                                    }
+                                    this.reputFromOffset += result.getSize() - readSize;
                                 }
                             }
                         }

store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java

@@ -22,7 +22,7 @@ public class DispatchRequest {
     private final String topic;
     private final int queueId;
     private final long commitLogOffset;
-    private final int msgSize;
+    private int msgSize;
     private final long tagsCode;
     private final long storeTimestamp;
     private final long consumeQueueOffset;
@@ -35,6 +35,8 @@ public class DispatchRequest {
     private final Map<String, String> propertiesMap;
     private byte[] bitMap;
 
+    private int bufferSize = -1;//the buffer size maybe larger than the msg size if the message is wrapped by something
+
     public DispatchRequest(
         final String topic,
         final int queueId,
@@ -156,4 +158,16 @@ public class DispatchRequest {
     public void setBitMap(byte[] bitMap) {
         this.bitMap = bitMap;
     }
+
+    public void setMsgSize(int msgSize) {
+        this.msgSize = msgSize;
+    }
+
+    public int getBufferSize() {
+        return bufferSize;
+    }
+
+    public void setBufferSize(int bufferSize) {
+        this.bufferSize = bufferSize;
+    }
 }

store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java

@@ -154,8 +154,8 @@ public class MappedFileQueue {
 
                 if (file.length() != this.mappedFileSize) {
                     log.warn(file + "\t" + file.length()
-                        + " length not matched message store config value, ignore it");
-                    return true;
+                        + " length not matched message store config value, please check it manually");
+                    return false;
                 }
 
                 try {

store/src/main/java/org/apache/rocketmq/store/MessageStore.java

@@ -21,6 +21,7 @@ import java.util.LinkedList;
 import java.util.Set;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.store.config.BrokerRole;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
 /**
@@ -366,4 +367,10 @@ public interface MessageStore {
      * @return BrokerStatsManager.
      */
     BrokerStatsManager getBrokerStatsManager();
+
+    /**
+     * handle
+     * @param brokerRole
+     */
+    void handleScheduleMessageService(BrokerRole brokerRole);
 }

store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java

@@ -48,9 +48,9 @@ public class SelectMappedBufferResult {
         this.byteBuffer.limit(this.size);
     }
 
-    public MappedFile getMappedFile() {
+   /* public MappedFile getMappedFile() {
         return mappedFile;
-    }
+    }*/
 
 //    @Override
 //    protected void finalize() {

store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java

@@ -143,6 +143,11 @@ public class MessageStoreConfig {
     private int transientStorePoolSize = 5;
     private boolean fastFailIfNoBufferInStorePool = false;
 
+    private boolean enableDLegerCommitLog = false;
+    private String dLegerGroup;
+    private String dLegerPeers;
+    private String dLegerSelfId;
+
     public boolean isDebugLockEnable() {
         return debugLockEnable;
     }
@@ -666,4 +671,35 @@ public class MessageStoreConfig {
         this.commitCommitLogThoroughInterval = commitCommitLogThoroughInterval;
     }
 
+    public String getdLegerGroup() {
+        return dLegerGroup;
+    }
+
+    public void setdLegerGroup(String dLegerGroup) {
+        this.dLegerGroup = dLegerGroup;
+    }
+
+    public String getdLegerPeers() {
+        return dLegerPeers;
+    }
+
+    public void setdLegerPeers(String dLegerPeers) {
+        this.dLegerPeers = dLegerPeers;
+    }
+
+    public String getdLegerSelfId() {
+        return dLegerSelfId;
+    }
+
+    public void setdLegerSelfId(String dLegerSelfId) {
+        this.dLegerSelfId = dLegerSelfId;
+    }
+
+    public boolean isEnableDLegerCommitLog() {
+        return enableDLegerCommitLog;
+    }
+
+    public void setEnableDLegerCommitLog(boolean enableDLegerCommitLog) {
+        this.enableDLegerCommitLog = enableDLegerCommitLog;
+    }
 }

store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java

@@ -0,0 +1,747 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.dledger;
+
+import io.openmessaging.storage.dledger.AppendFuture;
+import io.openmessaging.storage.dledger.DLedgerConfig;
+import io.openmessaging.storage.dledger.DLedgerServer;
+import io.openmessaging.storage.dledger.entry.DLedgerEntry;
+import io.openmessaging.storage.dledger.protocol.AppendEntryRequest;
+import io.openmessaging.storage.dledger.protocol.AppendEntryResponse;
+import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode;
+import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
+import io.openmessaging.storage.dledger.store.file.MmapFile;
+import io.openmessaging.storage.dledger.store.file.MmapFileList;
+import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult;
+import io.openmessaging.storage.dledger.utils.DLedgerUtils;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.common.message.MessageAccessor;
+import org.apache.rocketmq.common.message.MessageConst;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExtBatch;
+import org.apache.rocketmq.common.sysflag.MessageSysFlag;
+import org.apache.rocketmq.store.AppendMessageResult;
+import org.apache.rocketmq.store.AppendMessageStatus;
+import org.apache.rocketmq.store.CommitLog;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.DispatchRequest;
+import org.apache.rocketmq.store.MappedFile;
+import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.SelectMappedBufferResult;
+import org.apache.rocketmq.store.StoreStatsService;
+import org.apache.rocketmq.store.schedule.ScheduleMessageService;
+
+/**
+ * Store all metadata downtime for recovery, data protection reliability
+ */
+public class DLedgerCommitLog extends CommitLog {
+    private final DLedgerServer dLedgerServer;
+    private final DLedgerConfig dLedgerConfig;
+    private final DLedgerMmapFileStore dLedgerFileStore;
+    private final MmapFileList dLedgerFileList;
+
+    //The id identifies the broker role, 0 means master, others means slave
+    private final int id;
+
+    private final MessageSerializer messageSerializer;
+    private volatile long beginTimeInDledgerLock = 0;
+
+    //This offset separate the old commitlog from dledger commitlog
+    private long dividedCommitlogOffset = -1;
+
+
+    private boolean isInrecoveringOldCommitlog = false;
+
+    public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) {
+        super(defaultMessageStore);
+        dLedgerConfig =  new DLedgerConfig();
+        dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable());
+        dLedgerConfig.setStoreType(DLedgerConfig.FILE);
+        dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId());
+        dLedgerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup());
+        dLedgerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers());
+        dLedgerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
+        dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog());
+        dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen());
+        dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1);
+        id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1;
+        dLedgerServer = new DLedgerServer(dLedgerConfig);
+        dLedgerFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();
+        DLedgerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> {
+            assert bodyOffset == DLedgerEntry.BODY_OFFSET;
+            buffer.position(buffer.position() + bodyOffset + MessageDecoder.PHY_POS_POSITION);
+            buffer.putLong(entry.getPos() + bodyOffset);
+        };
+        dLedgerFileStore.addAppendHook(appendHook);
+        dLedgerFileList = dLedgerFileStore.getDataFileList();
+        this.messageSerializer = new MessageSerializer(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize());
+
+    }
+
+    @Override
+    public boolean load() {
+        boolean result = super.load();
+        if (!result) {
+            return false;
+        }
+
+        return true;
+    }
+
+    private void refreshConfig() {
+        dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable());
+        dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen());
+        dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1);
+    }
+
+    private void disableDeleteDledger() {
+        dLedgerConfig.setEnableDiskForceClean(false);
+        dLedgerConfig.setFileReservedHours(24 * 365 * 10);
+    }
+
+    @Override
+    public void start() {
+        dLedgerServer.startup();
+    }
+
+    @Override
+    public void shutdown() {
+        dLedgerServer.shutdown();
+    }
+
+    @Override
+    public long flush() {
+        dLedgerFileStore.flush();
+        return dLedgerFileList.getFlushedWhere();
+    }
+
+    @Override
+    public long getMaxOffset() {
+        if (dLedgerFileStore.getCommittedPos() > 0) {
+            return dLedgerFileStore.getCommittedPos();
+        }
+        if (dLedgerFileList.getMinOffset() > 0) {
+            return dLedgerFileList.getMinOffset();
+        }
+        return 0;
+    }
+
+    @Override
+    public long getMinOffset() {
+        if (!mappedFileQueue.getMappedFiles().isEmpty()) {
+            return mappedFileQueue.getMinOffset();
+        }
+        return dLedgerFileList.getMinOffset();
+    }
+
+    @Override
+    public long getConfirmOffset() {
+        return this.getMaxOffset();
+    }
+
+    @Override
+    public void setConfirmOffset(long phyOffset) {
+        log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset);
+    }
+
+
+
+    @Override
+    public long remainHowManyDataToCommit() {
+        return dLedgerFileList.remainHowManyDataToCommit();
+    }
+
+    @Override
+    public long remainHowManyDataToFlush() {
+        return dLedgerFileList.remainHowManyDataToFlush();
+    }
+
+    @Override
+    public int deleteExpiredFile(
+        final long expiredTime,
+        final int deleteFilesInterval,
+        final long intervalForcibly,
+        final boolean cleanImmediately
+    ) {
+        if (mappedFileQueue.getMappedFiles().isEmpty()) {
+            refreshConfig();
+            //To prevent too much log in defaultMessageStore
+            return  Integer.MAX_VALUE;
+        } else {
+            disableDeleteDledger();
+        }
+        int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately);
+        if (count > 0 || mappedFileQueue.getMappedFiles().size() != 1) {
+            return count;
+        }
+        //the old logic will keep the last file, here to delete it
+        MappedFile mappedFile = mappedFileQueue.getLastMappedFile();
+        log.info("Try to delete the last old commitlog file {}", mappedFile.getFileName());
+        long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime;
+        if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) {
+            while (!mappedFile.destroy(10 * 1000)) {
+                DLedgerUtils.sleep(1000);
+            }
+            mappedFileQueue.getMappedFiles().remove(mappedFile);
+        }
+        return 1;
+    }
+
+
+    public SelectMappedBufferResult convertSbr(SelectMmapBufferResult sbr) {
+        if (sbr == null) {
+            return null;
+        } else {
+            return new DLedgerSelectMappedBufferResult(sbr);
+        }
+
+    }
+
+    public SelectMmapBufferResult truncate(SelectMmapBufferResult sbr) {
+        long committedPos = dLedgerFileStore.getCommittedPos();
+        if (sbr == null || sbr.getStartOffset() == committedPos) {
+            return null;
+        }
+        if (sbr.getStartOffset() + sbr.getSize() <= committedPos) {
+            return sbr;
+        } else {
+            sbr.setSize((int) (committedPos - sbr.getStartOffset()));
+            return sbr;
+        }
+    }
+
+    @Override
+    public SelectMappedBufferResult getData(final long offset) {
+        if (offset < dividedCommitlogOffset) {
+            return super.getData(offset);
+        }
+        return this.getData(offset, offset == 0);
+    }
+
+
+    @Override
+    public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) {
+        if (offset < dividedCommitlogOffset) {
+            return super.getData(offset, returnFirstOnNotFound);
+        }
+        if (offset >= dLedgerFileStore.getCommittedPos()) {
+            return null;
+        }
+        int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData();
+        MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound);
+        if (mappedFile != null) {
+            int pos = (int) (offset % mappedFileSize);
+            SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos);
+            return  convertSbr(truncate(sbr));
+        }
+
+        return null;
+    }
+
+    private void recover(long maxPhyOffsetOfConsumeQueue) {
+        dLedgerFileStore.load();
+        if (dLedgerFileList.getMappedFiles().size() > 0) {
+            dLedgerFileStore.recover();
+            dividedCommitlogOffset = dLedgerFileList.getFirstMappedFile().getFileFromOffset();
+            MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+            if (mappedFile != null) {
+                disableDeleteDledger();
+            }
+            long maxPhyOffset = dLedgerFileList.getMaxWrotePosition();
+            // Clear ConsumeQueue redundant data
+            if (maxPhyOffsetOfConsumeQueue >= maxPhyOffset) {
+                log.warn("[TruncateCQ]maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, maxPhyOffset);
+                this.defaultMessageStore.truncateDirtyLogicFiles(maxPhyOffset);
+            }
+            return;
+        }
+        //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog
+        isInrecoveringOldCommitlog = true;
+        //No need the abnormal recover
+        super.recoverNormally(maxPhyOffsetOfConsumeQueue);
+        isInrecoveringOldCommitlog = false;
+        MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile();
+        if (mappedFile == null) {
+            return;
+        }
+        ByteBuffer byteBuffer =  mappedFile.sliceByteBuffer();
+        byteBuffer.position(mappedFile.getWrotePosition());
+        boolean needWriteMagicCode = true;
+        // 1 TOTAL SIZE
+        byteBuffer.getInt(); //size
+        int magicCode = byteBuffer.getInt();
+        if (magicCode == CommitLog.BLANK_MAGIC_CODE) {
+            needWriteMagicCode = false;
+        } else {
+            log.info("Recover old commitlog found a illegal magic code={}", magicCode);
+        }
+        dLedgerConfig.setEnableDiskForceClean(false);
+        dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize();
+        log.info("Recover old commitlog needWriteMagicCode={} pos={} file={} dividedCommitlogOffset={}", needWriteMagicCode, mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(), mappedFile.getFileName(), dividedCommitlogOffset);
+        if (needWriteMagicCode) {
+            byteBuffer.position(mappedFile.getWrotePosition());
+            byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition());
+            byteBuffer.putInt(BLANK_MAGIC_CODE);
+            mappedFile.flush(0);
+        }
+        mappedFile.setWrotePosition(mappedFile.getFileSize());
+        mappedFile.setCommittedPosition(mappedFile.getFileSize());
+        mappedFile.setFlushedPosition(mappedFile.getFileSize());
+        dLedgerFileList.getLastMappedFile(dividedCommitlogOffset);
+        log.info("Will set the initial commitlog offset={} for dledger", dividedCommitlogOffset);
+    }
+
+    @Override
+    public void recoverNormally(long maxPhyOffsetOfConsumeQueue) {
+        recover(maxPhyOffsetOfConsumeQueue);
+    }
+
+    @Override
+    public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue)  {
+        recover(maxPhyOffsetOfConsumeQueue);
+    }
+
+    @Override
+    public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC) {
+        return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true);
+    }
+
+    @Override
+    public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC,
+        final boolean readBody) {
+        if (isInrecoveringOldCommitlog) {
+            return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
+        }
+        try {
+            int bodyOffset = DLedgerEntry.BODY_OFFSET;
+            int pos = byteBuffer.position();
+            int magic =  byteBuffer.getInt();
+            //In dledger, this field is size, it must be gt 0, so it could prevent collision
+            int magicOld =  byteBuffer.getInt();
+            if (magicOld == CommitLog.BLANK_MAGIC_CODE || magicOld == CommitLog.MESSAGE_MAGIC_CODE) {
+                byteBuffer.position(pos);
+                return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
+            }
+            if (magic == MmapFileList.BLANK_MAGIC_CODE) {
+                return new DispatchRequest(0, true);
+            }
+            byteBuffer.position(pos + bodyOffset);
+            DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
+            if (dispatchRequest.isSuccess()) {
+                dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
+            } else if (dispatchRequest.getMsgSize() > 0) {
+                dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
+            }
+            return dispatchRequest;
+        } catch (Throwable ignored) {
+        }
+
+        return new DispatchRequest(-1, false /* success */);
+    }
+
+    @Override
+    public boolean resetOffset(long offset) {
+        //currently, it seems resetOffset has no use
+        return false;
+    }
+
+    @Override
+    public long getBeginTimeInLock() {
+        return beginTimeInDledgerLock;
+    }
+
+    @Override
+    public PutMessageResult putMessage(final MessageExtBrokerInner msg) {
+        // Set the storage time
+        msg.setStoreTimestamp(System.currentTimeMillis());
+        // Set the message body BODY CRC (consider the most appropriate setting
+        // on the client)
+        msg.setBodyCRC(UtilAll.crc32(msg.getBody()));
+
+        StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService();
+
+        String topic = msg.getTopic();
+        int queueId = msg.getQueueId();
+
+        //should be consistent with the old version
+        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
+        if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE
+            || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) {
+            // Delay Delivery
+            if (msg.getDelayTimeLevel() > 0) {
+                if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) {
+                    msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel());
+                }
+
+                topic = ScheduleMessageService.SCHEDULE_TOPIC;
+                queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel());
+
+                // Backup real topic, queueId
+                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic());
+                MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId()));
+                msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
+
+                msg.setTopic(topic);
+                msg.setQueueId(queueId);
+            }
+        }
+
+        // Back to Results
+        AppendMessageResult appendResult;
+        AppendFuture<AppendEntryResponse> dledgerFuture;
+        EncodeResult encodeResult;
+
+        putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
+        long eclipseTimeInLock;
+        long queueOffset;
+        try {
+            beginTimeInDledgerLock =  this.defaultMessageStore.getSystemClock().now();
+            encodeResult = this.messageSerializer.serialize(msg);
+            queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey);
+            if (encodeResult.status  != AppendMessageStatus.PUT_OK) {
+                return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status));
+            }
+            AppendEntryRequest request = new AppendEntryRequest();
+            request.setGroup(dLedgerConfig.getGroup());
+            request.setRemoteId(dLedgerServer.getMemberState().getSelfId());
+            request.setBody(encodeResult.data);
+            dledgerFuture = (AppendFuture<AppendEntryResponse>) dLedgerServer.handleAppend(request);
+            if (dledgerFuture.getPos() == -1) {
+                return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
+            }
+            long wroteOffset =  dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET;
+            ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH);
+            String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset);
+            eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock;
+            appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, eclipseTimeInLock);
+            switch (tranType) {
+                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
+                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
+                    break;
+                case MessageSysFlag.TRANSACTION_NOT_TYPE:
+                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
+                    // The next update ConsumeQueue information
+                    DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1);
+                    break;
+                default:
+                    break;
+            }
+        } catch (Exception e) {
+            log.error("Put message error", e);
+            return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR));
+        } finally {
+            beginTimeInDledgerLock = 0;
+            putMessageLock.unlock();
+        }
+
+        if (eclipseTimeInLock > 500) {
+            log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, appendResult);
+        }
+
+        PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR;
+        try {
+            AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS);
+            switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) {
+                case SUCCESS:
+                    putMessageStatus = PutMessageStatus.PUT_OK;
+                    break;
+                case INCONSISTENT_LEADER:
+                case NOT_LEADER:
+                case LEADER_NOT_READY:
+                case DISK_FULL:
+                    putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE;
+                    break;
+                case WAIT_QUORUM_ACK_TIMEOUT:
+                    //Do not return flush_slave_timeout to the client, for the ons client will ignore it.
+                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
+                    break;
+                case LEADER_PENDING_FULL:
+                    putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY;
+                    break;
+            }
+        } catch (Throwable t) {
+            log.error("Failed to get dledger append result", t);
+        }
+
+        PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult);
+        if (putMessageStatus == PutMessageStatus.PUT_OK) {
+            // Statistics
+            storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet();
+            storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(appendResult.getWroteBytes());
+        }
+        return putMessageResult;
+    }
+
+    @Override
+    public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) {
+        return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null);
+    }
+
+
+
+    @Override
+    public SelectMappedBufferResult getMessage(final long offset, final int size) {
+        if (offset < dividedCommitlogOffset) {
+            return super.getMessage(offset, size);
+        }
+        int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData();
+        MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0);
+        if (mappedFile != null) {
+            int pos = (int) (offset % mappedFileSize);
+            return  convertSbr(mappedFile.selectMappedBuffer(pos, size));
+        }
+        return null;
+    }
+
+    @Override
+    public long rollNextFile(final long offset) {
+        int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog();
+        return offset + mappedFileSize - offset % mappedFileSize;
+    }
+
+    @Override
+    public HashMap<String, Long> getTopicQueueTable() {
+        return topicQueueTable;
+    }
+
+    @Override
+    public void setTopicQueueTable(HashMap<String, Long> topicQueueTable) {
+        this.topicQueueTable = topicQueueTable;
+    }
+
+    @Override
+    public void destroy() {
+        super.destroy();
+        dLedgerFileList.destroy();
+    }
+
+    @Override
+    public boolean appendData(long startOffset, byte[] data) {
+        //the old ha service will invoke method, here to prevent it
+        return false;
+    }
+
+    @Override
+    public void checkSelf() {
+        dLedgerFileList.checkSelf();
+    }
+
+    @Override
+    public long lockTimeMills() {
+        long diff = 0;
+        long begin = this.beginTimeInDledgerLock;
+        if (begin > 0) {
+            diff = this.defaultMessageStore.now() - begin;
+        }
+
+        if (diff < 0) {
+            diff = 0;
+        }
+
+        return diff;
+    }
+
+    class EncodeResult {
+        private String queueOffsetKey;
+        private byte[] data;
+        private AppendMessageStatus status;
+        public EncodeResult(AppendMessageStatus status, byte[] data, String queueOffsetKey) {
+            this.data = data;
+            this.status = status;
+            this.queueOffsetKey = queueOffsetKey;
+        }
+    }
+
+    class MessageSerializer {
+        // File at the end of the minimum fixed length empty
+        private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4;
+        private final ByteBuffer msgIdMemory;
+        // Store the message content
+        private final ByteBuffer msgStoreItemMemory;
+        // The maximum length of the message
+        private final int maxMessageSize;
+        // Build Message Key
+        private final StringBuilder keyBuilder = new StringBuilder();
+
+        private final StringBuilder msgIdBuilder = new StringBuilder();
+
+        private final ByteBuffer hostHolder = ByteBuffer.allocate(8);
+
+        MessageSerializer(final int size) {
+            this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH);
+            this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH);
+            this.maxMessageSize = size;
+        }
+
+        public ByteBuffer getMsgStoreItemMemory() {
+            return msgStoreItemMemory;
+        }
+
+        public EncodeResult serialize(final MessageExtBrokerInner msgInner) {
+            // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>
+
+            // PHY OFFSET
+            long wroteOffset = 0;
+
+            this.resetByteBuffer(hostHolder, 8);
+            // Record ConsumeQueue information
+            keyBuilder.setLength(0);
+            keyBuilder.append(msgInner.getTopic());
+            keyBuilder.append('-');
+            keyBuilder.append(msgInner.getQueueId());
+            String key = keyBuilder.toString();
+
+            Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key);
+            if (null == queueOffset) {
+                queueOffset = 0L;
+                DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset);
+            }
+
+            // Transaction messages that require special handling
+            final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag());
+            switch (tranType) {
+                // Prepared and Rollback message is not consumed, will not enter the
+                // consumer queuec
+                case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
+                case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
+                    queueOffset = 0L;
+                    break;
+                case MessageSysFlag.TRANSACTION_NOT_TYPE:
+                case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
+                default:
+                    break;
+            }
+
+            /**
+             * Serialize message
+             */
+            final byte[] propertiesData =
+                msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8);
+
+            final int propertiesLength = propertiesData == null ? 0 : propertiesData.length;
+
+            if (propertiesLength > Short.MAX_VALUE) {
+                log.warn("putMessage message properties length too long. length={}", propertiesData.length);
+                return new EncodeResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED, null, key);
+            }
+
+            final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8);
+            final int topicLength = topicData.length;
+
+            final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length;
+
+            final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength);
+
+            // Exceeds the maximum message
+            if (msgLen > this.maxMessageSize) {
+                DLedgerCommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength
+                    + ", maxMessageSize: " + this.maxMessageSize);
+                return new EncodeResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED, null, key);
+            }
+            // Initialization of storage space
+            this.resetByteBuffer(msgStoreItemMemory, msgLen);
+            // 1 TOTALSIZE
+            this.msgStoreItemMemory.putInt(msgLen);
+            // 2 MAGICCODE
+            this.msgStoreItemMemory.putInt(DLedgerCommitLog.MESSAGE_MAGIC_CODE);
+            // 3 BODYCRC
+            this.msgStoreItemMemory.putInt(msgInner.getBodyCRC());
+            // 4 QUEUEID
+            this.msgStoreItemMemory.putInt(msgInner.getQueueId());
+            // 5 FLAG
+            this.msgStoreItemMemory.putInt(msgInner.getFlag());
+            // 6 QUEUEOFFSET
+            this.msgStoreItemMemory.putLong(queueOffset);
+            // 7 PHYSICALOFFSET
+            this.msgStoreItemMemory.putLong(wroteOffset);
+            // 8 SYSFLAG
+            this.msgStoreItemMemory.putInt(msgInner.getSysFlag());
+            // 9 BORNTIMESTAMP
+            this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp());
+            // 10 BORNHOST
+            this.resetByteBuffer(hostHolder, 8);
+            this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder));
+            // 11 STORETIMESTAMP
+            this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp());
+            // 12 STOREHOSTADDRESS
+            this.resetByteBuffer(hostHolder, 8);
+            this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder));
+            //this.msgBatchMemory.put(msgInner.getStoreHostBytes());
+            // 13 RECONSUMETIMES
+            this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes());
+            // 14 Prepared Transaction Offset
+            this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset());
+            // 15 BODY
+            this.msgStoreItemMemory.putInt(bodyLength);
+            if (bodyLength > 0) {
+                this.msgStoreItemMemory.put(msgInner.getBody());
+            }
+            // 16 TOPIC
+            this.msgStoreItemMemory.put((byte) topicLength);
+            this.msgStoreItemMemory.put(topicData);
+            // 17 PROPERTIES
+            this.msgStoreItemMemory.putShort((short) propertiesLength);
+            if (propertiesLength > 0) {
+                this.msgStoreItemMemory.put(propertiesData);
+            }
+            byte[] data = new byte[msgLen];
+            this.msgStoreItemMemory.clear();
+            this.msgStoreItemMemory.get(data);
+            return new EncodeResult(AppendMessageStatus.PUT_OK, data, key);
+        }
+
+        private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) {
+            byteBuffer.flip();
+            byteBuffer.limit(limit);
+        }
+
+    }
+
+    public static class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult {
+
+        private SelectMmapBufferResult sbr;
+        public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) {
+            super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null);
+            this.sbr = sbr;
+        }
+
+        public synchronized void release() {
+            super.release();
+            if (sbr != null) {
+                sbr.release();
+            }
+        }
+
+    }
+
+    public DLedgerServer getdLedgerServer() {
+        return dLedgerServer;
+    }
+
+    public int getId() {
+        return id;
+    }
+
+    public long getDividedCommitlogOffset() {
+        return dividedCommitlogOffset;
+    }
+}

store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java

@@ -90,7 +90,7 @@ public class HAConnection {
             this.selector = RemotingUtil.openSelector();
             this.socketChannel = socketChannel;
             this.socketChannel.register(this.selector, SelectionKey.OP_READ);
-            this.thread.setDaemon(true);
+            this.setDaemon(true);
         }
 
         @Override
@@ -205,7 +205,7 @@ public class HAConnection {
             this.selector = RemotingUtil.openSelector();
             this.socketChannel = socketChannel;
             this.socketChannel.register(this.selector, SelectionKey.OP_WRITE);
-            this.thread.setDaemon(true);
+            this.setDaemon(true);
         }
 
         @Override

store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java

@@ -19,11 +19,11 @@ package org.apache.rocketmq.store.schedule;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.rocketmq.common.ConfigManager;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.common.constant.LoggerName;
@@ -38,6 +38,7 @@ import org.apache.rocketmq.store.ConsumeQueue;
 import org.apache.rocketmq.store.ConsumeQueueExt;
 import org.apache.rocketmq.store.DefaultMessageStore;
 import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageStore;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.PutMessageStatus;
 import org.apache.rocketmq.store.SelectMappedBufferResult;
@@ -56,15 +57,15 @@ public class ScheduleMessageService extends ConfigManager {
 
     private final ConcurrentMap<Integer /* level */, Long/* offset */> offsetTable =
         new ConcurrentHashMap<Integer, Long>(32);
-
-    private final Timer timer = new Timer("ScheduleMessageTimerThread", true);
-
     private final DefaultMessageStore defaultMessageStore;
-
+    private final AtomicBoolean started = new AtomicBoolean(false);
+    private Timer timer;
+    private MessageStore writeMessageStore;
     private int maxDelayLevel;
 
     public ScheduleMessageService(final DefaultMessageStore defaultMessageStore) {
         this.defaultMessageStore = defaultMessageStore;
+        this.writeMessageStore = defaultMessageStore;
     }
 
     public static int queueId2DelayLevel(final int queueId) {
@@ -75,10 +76,18 @@ public class ScheduleMessageService extends ConfigManager {
         return delayLevel - 1;
     }
 
+    /**
+     * @param writeMessageStore
+     *     the writeMessageStore to set
+     */
+    public void setWriteMessageStore(MessageStore writeMessageStore) {
+        this.writeMessageStore = writeMessageStore;
+    }
+
     public void buildRunningStats(HashMap<String, String> stats) {
-        Iterator<Entry<Integer, Long>> it = this.offsetTable.entrySet().iterator();
+        Iterator<Map.Entry<Integer, Long>> it = this.offsetTable.entrySet().iterator();
         while (it.hasNext()) {
-            Entry<Integer, Long> next = it.next();
+            Map.Entry<Integer, Long> next = it.next();
             int queueId = delayLevel2QueueId(next.getKey());
             long delayOffset = next.getValue();
             long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(SCHEDULE_TOPIC, queueId);
@@ -102,35 +111,45 @@ public class ScheduleMessageService extends ConfigManager {
     }
 
     public void start() {
+        if (started.compareAndSet(false, true)) {
+            this.timer = new Timer("ScheduleMessageTimerThread", true);
+            for (Map.Entry<Integer, Long> entry : this.delayLevelTable.entrySet()) {
+                Integer level = entry.getKey();
+                Long timeDelay = entry.getValue();
+                Long offset = this.offsetTable.get(level);
+                if (null == offset) {
+                    offset = 0L;
+                }
 
-        for (Map.Entry<Integer, Long> entry : this.delayLevelTable.entrySet()) {
-            Integer level = entry.getKey();
-            Long timeDelay = entry.getValue();
-            Long offset = this.offsetTable.get(level);
-            if (null == offset) {
-                offset = 0L;
-            }
-
-            if (timeDelay != null) {
-                this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME);
-            }
-        }
-
-        this.timer.scheduleAtFixedRate(new TimerTask() {
-
-            @Override
-            public void run() {
-                try {
-                    ScheduleMessageService.this.persist();
-                } catch (Throwable e) {
-                    log.error("scheduleAtFixedRate flush exception", e);
+                if (timeDelay != null) {
+                    this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME);
                 }
             }
-        }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval());
+
+            this.timer.scheduleAtFixedRate(new TimerTask() {
+
+                @Override
+                public void run() {
+                    try {
+                        if (started.get()) ScheduleMessageService.this.persist();
+                    } catch (Throwable e) {
+                        log.error("scheduleAtFixedRate flush exception", e);
+                    }
+                }
+            }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval());
+        }
     }
 
     public void shutdown() {
-        this.timer.cancel();
+        if (this.started.compareAndSet(true, false)) {
+            if (null != this.timer)
+                this.timer.cancel();
+        }
+
+    }
+
+    public boolean isStarted() {
+        return started.get();
     }
 
     public int getMaxDelayLevel() {
@@ -214,7 +233,9 @@ public class ScheduleMessageService extends ConfigManager {
         @Override
         public void run() {
             try {
-                this.executeOnTimeup();
+                if (isStarted()) {
+                    this.executeOnTimeup();
+                }
             } catch (Exception e) {
                 // XXX: warn and notify me
                 log.error("ScheduleMessageService, executeOnTimeup exception", e);
@@ -285,7 +306,7 @@ public class ScheduleMessageService extends ConfigManager {
                                     try {
                                         MessageExtBrokerInner msgInner = this.messageTimeup(msgExt);
                                         PutMessageResult putMessageResult =
-                                            ScheduleMessageService.this.defaultMessageStore
+                                            ScheduleMessageService.this.writeMessageStore
                                                 .putMessage(msgInner);
 
                                         if (putMessageResult != null

store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java

@@ -30,9 +30,10 @@ import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.message.MessageDecoder;
 import org.apache.rocketmq.store.config.MessageStoreConfig;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
-import static org.assertj.core.api.Assertions.assertThat;
 import org.junit.Test;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 public class ConsumeQueueTest {
 
     private static final String msg = "Once, there was a chance for me!";

store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java

@@ -42,9 +42,9 @@ public class DefaultMessageStoreShutDownTest {
     public void init() throws Exception {
         messageStore = spy(buildMessageStore());
         boolean load = messageStore.load();
-        when(messageStore.dispatchBehindBytes()).thenReturn(100L);
         assertTrue(load);
         messageStore.start();
+        when(messageStore.dispatchBehindBytes()).thenReturn(100L);
     }
 
     @Test

store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.junit.After;
+
+public class StoreTestBase {
+
+    private int QUEUE_TOTAL = 100;
+    private AtomicInteger QueueId = new AtomicInteger(0);
+    private SocketAddress BornHost = new InetSocketAddress("127.0.0.1", 8123);
+    private SocketAddress StoreHost = BornHost;
+    private byte[] MessageBody = new byte[1024];
+
+    protected Set<String> baseDirs = new HashSet<>();
+
+    private static AtomicInteger port = new AtomicInteger(30000);
+
+    public static synchronized int nextPort() {
+        return port.addAndGet(5);
+    }
+
+    protected MessageExtBrokerInner buildMessage() {
+        MessageExtBrokerInner msg = new MessageExtBrokerInner();
+        msg.setTopic("StoreTest");
+        msg.setTags("TAG1");
+        msg.setKeys("Hello");
+        msg.setBody(MessageBody);
+        msg.setKeys(String.valueOf(System.currentTimeMillis()));
+        msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL);
+        msg.setSysFlag(0);
+        msg.setBornTimestamp(System.currentTimeMillis());
+        msg.setStoreHost(StoreHost);
+        msg.setBornHost(BornHost);
+        return msg;
+    }
+
+    public static String createBaseDir() {
+        String baseDir = System.getProperty("user.home") + File.separator + "unitteststore" + File.separator + UUID.randomUUID();
+        final File file = new File(baseDir);
+        if (file.exists()) {
+            System.exit(1);
+        }
+        return baseDir;
+    }
+
+    public static boolean makeSureFileExists(String fileName) throws Exception {
+        File file = new File(fileName);
+        MappedFile.ensureDirOK(file.getParent());
+        return file.createNewFile();
+    }
+
+
+    public static void deleteFile(String fileName) {
+        deleteFile(new File(fileName));
+    }
+
+    public static void deleteFile(File file) {
+        UtilAll.deleteFile(file);
+    }
+
+    @After
+    public void clear() {
+        for (String baseDir : baseDirs) {
+            deleteFile(baseDir);
+        }
+    }
+
+}

store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java

@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.dledger;
+
+import io.openmessaging.storage.dledger.DLedgerServer;
+import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore;
+import io.openmessaging.storage.dledger.store.file.MmapFileList;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.GetMessageStatus;
+import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DLedgerCommitlogTest extends MessageStoreTestBase {
+
+
+    @Test
+    public void testTruncateCQ() throws Exception {
+        String base =  createBaseDir();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+        String topic = UUID.randomUUID().toString();
+        {
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
+            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();
+            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();
+            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();
+            Thread.sleep(2000);
+            doPutMessages(messageStore, topic, 0, 2000, 0);
+            Thread.sleep(100);
+            Assert.assertEquals(24, mmapFileList.getMappedFiles().size());
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(2000, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            doGetMessages(messageStore, topic, 0, 2000, 0);
+            messageStore.shutdown();
+        }
+
+        {
+            //Abnormal recover, left some commitlogs
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 4);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
+            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();
+            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();
+            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();
+            Thread.sleep(1000);
+            Assert.assertEquals(20, mmapFileList.getMappedFiles().size());
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1700, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            doGetMessages(messageStore, topic, 0, 1700, 0);
+            messageStore.shutdown();
+        }
+        {
+            //Abnormal recover, left none commitlogs
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 20);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog();
+            DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer();
+            DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore();
+            MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList();
+            Thread.sleep(1000);
+            Assert.assertEquals(0, mmapFileList.getMappedFiles().size());
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            messageStore.shutdown();
+        }
+    }
+
+
+
+    @Test
+    public void testRecover() throws Exception {
+        String base =  createBaseDir();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+        String topic = UUID.randomUUID().toString();
+        {
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0);
+            Thread.sleep(1000);
+            doPutMessages(messageStore, topic, 0, 1000, 0);
+            Thread.sleep(100);
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            doGetMessages(messageStore, topic, 0, 1000, 0);
+            messageStore.shutdown();
+        }
+
+        {
+            //normal recover
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0);
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            doGetMessages(messageStore, topic, 0, 1000, 0);
+            messageStore.shutdown();
+        }
+
+        {
+            //abnormal recover
+            DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0);
+            Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+            doGetMessages(messageStore, topic, 0, 1000, 0);
+            messageStore.shutdown();
+        }
+    }
+
+
+
+    @Test
+    public void testPutAndGetMessage() throws Exception {
+        String base =  createBaseDir();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+        DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0);
+        Thread.sleep(1000);
+        String topic = UUID.randomUUID().toString();
+
+        List<PutMessageResult> results = new ArrayList<>();
+        for (int i = 0; i < 10; i++) {
+            MessageExtBrokerInner msgInner =  buildMessage();
+            msgInner.setTopic(topic);
+            msgInner.setQueueId(0);
+            PutMessageResult putMessageResult = messageStore.putMessage(msgInner);
+            results.add(putMessageResult);
+            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());
+            Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset());
+        }
+        Thread.sleep(100);
+        Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0));
+        Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0));
+        Assert.assertEquals(0, messageStore.dispatchBehindBytes());
+        GetMessageResult getMessageResult =  messageStore.getMessage("group", topic, 0, 0, 32, null);
+        Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus());
+
+        Assert.assertEquals(10, getMessageResult.getMessageBufferList().size());
+        Assert.assertEquals(10, getMessageResult.getMessageMapedList().size());
+
+        for (int i = 0; i < results.size(); i++) {
+            ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i);
+            MessageExt messageExt = MessageDecoder.decode(buffer);
+            Assert.assertEquals(i, messageExt.getQueueOffset());
+            Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId());
+            Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset());
+        }
+        messageStore.destroy();
+        messageStore.shutdown();
+    }
+
+
+    @Test
+    public void testCommittedPos() throws Exception {
+        String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort());
+        String group = UUID.randomUUID().toString();
+        DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false, 0);
+
+        String topic = UUID.randomUUID().toString();
+        MessageExtBrokerInner msgInner =  buildMessage();
+        msgInner.setTopic(topic);
+        msgInner.setQueueId(0);
+        PutMessageResult putMessageResult = leaderStore.putMessage(msgInner);
+        Assert.assertEquals(PutMessageStatus.OS_PAGECACHE_BUSY, putMessageResult.getPutMessageStatus());
+
+        Thread.sleep(1000);
+
+        Assert.assertEquals(0, leaderStore.getCommitLog().getMaxOffset());
+        Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0));
+
+
+        DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false, 0);
+        Thread.sleep(2000);
+
+        Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0));
+        Assert.assertEquals(1, followerStore.getMaxOffsetInQueue(topic, 0));
+        Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0);
+
+
+        leaderStore.destroy();
+        followerStore.destroy();
+
+        leaderStore.shutdown();
+        followerStore.shutdown();
+    }
+
+
+}

store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java

@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.dledger;
+
+import io.openmessaging.storage.dledger.DLedgerConfig;
+import io.openmessaging.storage.dledger.DLedgerServer;
+import java.io.File;
+import java.util.Arrays;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.GetMessageResult;
+import org.apache.rocketmq.store.MessageExtBrokerInner;
+import org.apache.rocketmq.store.MessageStore;
+import org.apache.rocketmq.store.PutMessageResult;
+import org.apache.rocketmq.store.PutMessageStatus;
+import org.apache.rocketmq.store.StoreTestBase;
+import org.apache.rocketmq.store.config.FlushDiskType;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.apache.rocketmq.store.stats.BrokerStatsManager;
+import org.junit.Assert;
+
+public class MessageStoreTestBase extends StoreTestBase {
+
+    protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort, int deleteFileNum) throws Exception {
+        System.setProperty("dledger.disk.ratio.check", "0.95");
+        System.setProperty("dledger.disk.ratio.clean", "0.95");
+        baseDirs.add(base);
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        storeConfig.setMapedFileSizeCommitLog(1024 * 100);
+        storeConfig.setMapedFileSizeConsumeQueue(1024);
+        storeConfig.setMaxHashSlotNum(100);
+        storeConfig.setMaxIndexNum(100 * 10);
+        storeConfig.setStorePathRootDir(base);
+        storeConfig.setStorePathCommitLog(base + File.separator + "commitlog");
+        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);
+
+        storeConfig.setEnableDLegerCommitLog(true);
+        storeConfig.setdLegerGroup(group);
+        storeConfig.setdLegerPeers(peers);
+        storeConfig.setdLegerSelfId(selfId);
+        DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig,  new BrokerStatsManager("DLedgerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {
+
+        }, new BrokerConfig());
+        DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer();
+        if (leaderId != null) {
+            dLegerServer.getdLedgerConfig().setEnableLeaderElector(false);
+            if (selfId.equals(leaderId)) {
+                dLegerServer.getMemberState().changeToLeader(-1);
+            } else {
+                dLegerServer.getMemberState().changeToFollower(-1, leaderId);
+            }
+
+        }
+        if (createAbort) {
+            String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir());
+            makeSureFileExists(fileName);
+        }
+        if (deleteFileNum > 0) {
+            DLedgerConfig config = dLegerServer.getdLedgerConfig();
+            if (deleteFileNum > 0) {
+                File dir = new File(config.getDataStorePath());
+                File[] files = dir.listFiles();
+                if (files != null) {
+                    Arrays.sort(files);
+                    for (int i = files.length - 1; i >= 0; i--) {
+                        File file = files[i];
+                        file.delete();
+                        if (files.length - i >= deleteFileNum) {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        Assert.assertTrue(defaultMessageStore.load());
+        defaultMessageStore.start();
+        return defaultMessageStore;
+    }
+
+
+    protected DefaultMessageStore createMessageStore(String base, boolean createAbort) throws Exception {
+        baseDirs.add(base);
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        storeConfig.setMapedFileSizeCommitLog(1024 * 100);
+        storeConfig.setMapedFileSizeConsumeQueue(1024);
+        storeConfig.setMaxHashSlotNum(100);
+        storeConfig.setMaxIndexNum(100 * 10);
+        storeConfig.setStorePathRootDir(base);
+        storeConfig.setStorePathCommitLog(base + File.separator + "commitlog");
+        storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);
+        DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig,  new BrokerStatsManager("CommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> {
+
+        }, new BrokerConfig());
+
+        if (createAbort) {
+            String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir());
+            makeSureFileExists(fileName);
+        }
+        Assert.assertTrue(defaultMessageStore.load());
+        defaultMessageStore.start();
+        return defaultMessageStore;
+    }
+
+    protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) {
+        for (int i = 0; i < num; i++) {
+            MessageExtBrokerInner msgInner = buildMessage();
+            msgInner.setTopic(topic);
+            msgInner.setQueueId(queueId);
+            PutMessageResult putMessageResult = messageStore.putMessage(msgInner);
+            Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus());
+            Assert.assertEquals(beginLogicsOffset + i, putMessageResult.getAppendMessageResult().getLogicsOffset());
+        }
+    }
+
+    protected void doGetMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) {
+        for (int i = 0; i < num; i++) {
+            GetMessageResult getMessageResult =  messageStore.getMessage("group", topic, queueId, beginLogicsOffset + i, 3, null);
+            Assert.assertNotNull(getMessageResult);
+            Assert.assertTrue(!getMessageResult.getMessageBufferList().isEmpty());
+            MessageExt messageExt = MessageDecoder.decode(getMessageResult.getMessageBufferList().get(0));
+            Assert.assertEquals(beginLogicsOffset + i, messageExt.getQueueOffset());
+            getMessageResult.release();
+        }
+    }
+
+}

store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java

@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.store.dledger;
+
+import java.util.UUID;
+import org.apache.rocketmq.store.DefaultMessageStore;
+import org.apache.rocketmq.store.StoreTestBase;
+import org.apache.rocketmq.store.config.StorePathConfigHelper;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MixCommitlogTest extends MessageStoreTestBase {
+
+
+
+    @Test
+    public void testFallBehindCQ() throws Exception {
+        String base =  createBaseDir();
+        String topic = UUID.randomUUID().toString();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+        {
+            DefaultMessageStore originalStore = createMessageStore(base, false);
+            doPutMessages(originalStore, topic, 0, 1000, 0);
+            Assert.assertEquals(11, originalStore.getMaxPhyOffset()/originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog());
+            Thread.sleep(500);
+            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, originalStore.dispatchBehindBytes());
+            doGetMessages(originalStore, topic, 0, 1000, 0);
+            originalStore.shutdown();
+        }
+        //delete the cq files
+        {
+            StoreTestBase.deleteFile(StorePathConfigHelper.getStorePathConsumeQueue(base));
+        }
+        {
+            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0);
+            Thread.sleep(2000);
+            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, dledgerStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());
+            doGetMessages(dledgerStore, topic, 0, 1000, 0);
+            doPutMessages(dledgerStore, topic, 0, 1000, 1000);
+            Thread.sleep(500);
+            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());
+            doGetMessages(dledgerStore, topic, 0, 2000, 0);
+            dledgerStore.shutdown();
+        }
+    }
+
+
+
+    @Test
+    public void testPutAndGet() throws Exception {
+        String base =  createBaseDir();
+        String topic = UUID.randomUUID().toString();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+
+        long dividedOffset;
+        {
+            DefaultMessageStore originalStore = createMessageStore(base, false);
+            doPutMessages(originalStore, topic, 0, 1000, 0);
+            Thread.sleep(500);
+            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, originalStore.dispatchBehindBytes());
+            dividedOffset = originalStore.getCommitLog().getMaxOffset();
+            dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog();
+            doGetMessages(originalStore, topic, 0, 1000, 0);
+            originalStore.shutdown();
+        }
+        {
+            DefaultMessageStore recoverOriginalStore = createMessageStore(base, true);
+            Thread.sleep(500);
+            Assert.assertEquals(0, recoverOriginalStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, recoverOriginalStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, recoverOriginalStore.dispatchBehindBytes());
+            doGetMessages(recoverOriginalStore, topic, 0, 1000, 0);
+            recoverOriginalStore.shutdown();
+        }
+        {
+            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog();
+            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());
+            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());
+            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());
+            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getMaxOffset());
+            Thread.sleep(2000);
+            doPutMessages(dledgerStore, topic, 0, 1000, 1000);
+            Thread.sleep(500);
+            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());
+            doGetMessages(dledgerStore, topic, 0, 2000, 0);
+            dledgerStore.shutdown();
+        }
+        {
+            DefaultMessageStore recoverDledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) recoverDledgerStore.getCommitLog();
+            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());
+            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());
+            Thread.sleep(2000);
+            doPutMessages(recoverDledgerStore, topic, 0, 1000, 2000);
+            Thread.sleep(500);
+            Assert.assertEquals(0, recoverDledgerStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(3000, recoverDledgerStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, recoverDledgerStore.dispatchBehindBytes());
+            doGetMessages(recoverDledgerStore, topic, 0, 3000, 0);
+            recoverDledgerStore.shutdown();
+        }
+    }
+
+    @Test
+    public void testDeleteExpiredFiles() throws Exception {
+        String base =  createBaseDir();
+        String topic = UUID.randomUUID().toString();
+        String peers = String.format("n0-localhost:%d", nextPort());
+        String group = UUID.randomUUID().toString();
+
+        long dividedOffset;
+        {
+            DefaultMessageStore originalStore = createMessageStore(base, false);
+            doPutMessages(originalStore, topic, 0, 1000, 0);
+            Thread.sleep(500);
+            Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, originalStore.dispatchBehindBytes());
+            dividedOffset = originalStore.getCommitLog().getMaxOffset();
+            dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog();
+            originalStore.shutdown();
+        }
+        long maxPhysicalOffset;
+        {
+            DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0);
+            DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog();
+            Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable());
+            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());
+            Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset());
+            Thread.sleep(2000);
+            doPutMessages(dledgerStore, topic, 0, 1000, 1000);
+            Thread.sleep(500);
+            Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0));
+            Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0));
+            Assert.assertEquals(0, dledgerStore.dispatchBehindBytes());
+            Assert.assertEquals(0, dledgerStore.getMinPhyOffset());
+            maxPhysicalOffset = dledgerStore.getMaxPhyOffset();
+            Assert.assertTrue(maxPhysicalOffset > 0);
+
+            doGetMessages(dledgerStore, topic, 0, 2000, 0);
+
+            for (int i = 0; i < 100; i++) {
+                dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true);
+            }
+            Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset());
+            Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset());
+            for (int i = 0; i < 100; i++) {
+                Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true));
+            }
+            Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset());
+            Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset());
+
+            Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable());
+            Assert.assertTrue(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());
+
+            //Test fresh
+            dledgerStore.getMessageStoreConfig().setCleanFileForciblyEnable(false);
+            for (int i = 0; i < 100; i++) {
+                Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true));
+            }
+            Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean());
+            doGetMessages(dledgerStore, topic, 0, 1000, 1000);
+            dledgerStore.shutdown();
+        }
+    }
+}

store/src/test/resources/logback-test.xml

@@ -28,7 +28,7 @@
         <appender-ref ref="STDOUT"/>
     </logger>
 
-    <root level="ERROR">
+    <root level="OFF">
         <appender-ref ref="STDOUT"/>
     </root>
 

test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java

@@ -17,6 +17,8 @@
 
 package org.apache.rocketmq.test.factory;
 
+import java.util.UUID;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
 import org.apache.rocketmq.client.consumer.MessageSelector;
 import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer;
 import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
@@ -60,4 +62,12 @@ public class ConsumerFactory {
         consumer.start();
         return consumer;
     }
+
+    public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception {
+        DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup);
+        defaultMQPullConsumer.setInstanceName(UUID.randomUUID().toString());
+        defaultMQPullConsumer.setNamesrvAddr(nsAddr);
+        defaultMQPullConsumer.start();
+        return defaultMQPullConsumer;
+    }
 }

test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.test.factory;
 
+import java.util.UUID;
 import org.apache.rocketmq.client.exception.MQClientException;
 import org.apache.rocketmq.client.producer.DefaultMQProducer;
 import org.apache.rocketmq.test.util.RandomUtil;
@@ -25,6 +26,7 @@ public class ProducerFactory {
 
     public static DefaultMQProducer getRMQProducer(String ns) {
         DefaultMQProducer producer = new DefaultMQProducer(RandomUtil.getStringByUUID());
+        producer.setInstanceName(UUID.randomUUID().toString());
         producer.setNamesrvAddr(ns);
         try {
             producer.start();

test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java

@@ -19,6 +19,7 @@ package org.apache.rocketmq.test.util;
 
 import java.util.HashMap;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.log4j.Logger;
 import org.apache.rocketmq.common.admin.TopicStatsTable;
 import org.apache.rocketmq.common.protocol.body.ClusterInfo;
@@ -40,6 +41,7 @@ public class MQAdmin {
         int queueNum, int waitTimeSec) {
         boolean createResult = false;
         DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt();
+        mqAdminExt.setInstanceName(UUID.randomUUID().toString());
         mqAdminExt.setNamesrvAddr(nameSrvAddr);
         try {
             mqAdminExt.start();

test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java

@@ -36,13 +36,13 @@ import org.apache.rocketmq.test.util.MQAdmin;
 import org.apache.rocketmq.test.util.MQRandomUtils;
 
 public class BaseConf {
-    protected static String nsAddr;
+    public static String nsAddr;
     protected static String broker1Name;
     protected static String broker2Name;
     protected static String clusterName;
     protected static int brokerNum;
     protected static int waitTime = 5;
-    protected static int consumeTime = 5 * 60 * 1000;
+    protected static int consumeTime = 2 * 60 * 1000;
     protected static NamesrvController namesrvController;
     protected static BrokerController brokerController1;
     protected static BrokerController brokerController2;

test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java

@@ -47,9 +47,14 @@ public class IntegrationTestBase {
     protected static final List<BrokerController> BROKER_CONTROLLERS = new ArrayList<>();
     protected static final List<NamesrvController> NAMESRV_CONTROLLERS = new ArrayList<>();
     protected static int topicCreateTime = 30 * 1000;
-    protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 256;
+    protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 100;
     protected static final int INDEX_NUM = 1000;
 
+    private static final AtomicInteger port = new AtomicInteger(40000);
+
+    public static synchronized int nextPort() {
+        return port.addAndGet(random.nextInt(10) + 10);
+    }
     protected static Random random = new Random();
 
     static {
@@ -87,7 +92,7 @@ public class IntegrationTestBase {
 
     }
 
-    private static String createBaseDir() {
+    public static String createBaseDir() {
         String baseDir = System.getProperty("user.home") + SEP + "unitteststore-" + UUID.randomUUID();
         final File file = new File(baseDir);
         if (file.exists()) {
@@ -105,14 +110,14 @@ public class IntegrationTestBase {
         namesrvConfig.setKvConfigPath(baseDir + SEP + "namesrv" + SEP + "kvConfig.json");
         namesrvConfig.setConfigStorePath(baseDir + SEP + "namesrv" + SEP + "namesrv.properties");
 
-        nameServerNettyServerConfig.setListenPort(9000 + random.nextInt(1000));
+        nameServerNettyServerConfig.setListenPort(nextPort());
         NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig);
         try {
             Assert.assertTrue(namesrvController.initialize());
             logger.info("Name Server Start:{}", nameServerNettyServerConfig.getListenPort());
             namesrvController.start();
         } catch (Exception e) {
-            logger.info("Name Server start failed");
+            logger.info("Name Server start failed", e);
             System.exit(1);
         }
         NAMESRV_CONTROLLERS.add(namesrvController);
@@ -123,8 +128,6 @@ public class IntegrationTestBase {
     public static BrokerController createAndStartBroker(String nsAddr) {
         String baseDir = createBaseDir();
         BrokerConfig brokerConfig = new BrokerConfig();
-        NettyServerConfig nettyServerConfig = new NettyServerConfig();
-        NettyClientConfig nettyClientConfig = new NettyClientConfig();
         MessageStoreConfig storeConfig = new MessageStoreConfig();
         brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.getAndIncrement());
         brokerConfig.setBrokerIP1("127.0.0.1");
@@ -132,18 +135,25 @@ public class IntegrationTestBase {
         brokerConfig.setEnablePropertyFilter(true);
         storeConfig.setStorePathRootDir(baseDir);
         storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
-        storeConfig.setHaListenPort(8000 + random.nextInt(1000));
         storeConfig.setMapedFileSizeCommitLog(COMMIT_LOG_SIZE);
         storeConfig.setMaxIndexNum(INDEX_NUM);
         storeConfig.setMaxHashSlotNum(INDEX_NUM * 4);
-        nettyServerConfig.setListenPort(10000 + random.nextInt(1000));
+        return createAndStartBroker(storeConfig, brokerConfig);
+
+    }
+
+    public static BrokerController createAndStartBroker(MessageStoreConfig storeConfig, BrokerConfig brokerConfig) {
+        NettyServerConfig nettyServerConfig = new NettyServerConfig();
+        NettyClientConfig nettyClientConfig = new NettyClientConfig();
+        nettyServerConfig.setListenPort(nextPort());
+        storeConfig.setHaListenPort(nextPort());
         BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig);
         try {
             Assert.assertTrue(brokerController.initialize());
             logger.info("Broker Start name:{} addr:{}", brokerConfig.getBrokerName(), brokerController.getBrokerAddr());
             brokerController.start();
-        } catch (Exception e) {
-            logger.info("Broker start failed");
+        } catch (Throwable t) {
+            logger.error("Broker start failed, will exit", t);
             System.exit(1);
         }
         BROKER_CONTROLLERS.add(brokerController);
@@ -179,15 +189,7 @@ public class IntegrationTestBase {
         if (!file.exists()) {
             return;
         }
-        if (file.isFile()) {
-            file.delete();
-        } else if (file.isDirectory()) {
-            File[] files = file.listFiles();
-            for (File file1 : files) {
-                deleteFile(file1);
-            }
-            file.delete();
-        }
+        UtilAll.deleteFile(file);
     }
 
 }

test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.rocketmq.test.base.dledger;
+
+import java.util.UUID;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.producer.DefaultMQProducer;
+import org.apache.rocketmq.client.producer.SendResult;
+import org.apache.rocketmq.client.producer.SendStatus;
+import org.apache.rocketmq.common.BrokerConfig;
+import org.apache.rocketmq.common.message.Message;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.store.config.BrokerRole;
+import org.apache.rocketmq.store.config.MessageStoreConfig;
+import org.apache.rocketmq.test.base.BaseConf;
+import org.apache.rocketmq.test.base.IntegrationTestBase;
+import org.apache.rocketmq.test.factory.ConsumerFactory;
+import org.apache.rocketmq.test.factory.ProducerFactory;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.rocketmq.test.base.IntegrationTestBase.nextPort;
+import static sun.util.locale.BaseLocale.SEP;
+
+public class DLedgerProduceAndConsumeIT {
+
+    public BrokerConfig buildBrokerConfig(String cluster, String brokerName) {
+        BrokerConfig brokerConfig =  new BrokerConfig();
+        brokerConfig.setBrokerClusterName(cluster);
+        brokerConfig.setBrokerName(brokerName);
+        brokerConfig.setBrokerIP1("127.0.0.1");
+        brokerConfig.setNamesrvAddr(BaseConf.nsAddr);
+        return brokerConfig;
+    }
+
+    public MessageStoreConfig buildStoreConfig(String brokerName, String peers, String selfId) {
+        MessageStoreConfig storeConfig = new MessageStoreConfig();
+        String baseDir =  IntegrationTestBase.createBaseDir();
+        storeConfig.setStorePathRootDir(baseDir);
+        storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog");
+        storeConfig.setHaListenPort(nextPort());
+        storeConfig.setMapedFileSizeCommitLog(10 * 1024 * 1024);
+        storeConfig.setEnableDLegerCommitLog(true);
+        storeConfig.setdLegerGroup(brokerName);
+        storeConfig.setdLegerSelfId(selfId);
+        storeConfig.setdLegerPeers(peers);
+        return storeConfig;
+    }
+
+    @Test
+    public void testProduceAndConsume() throws Exception {
+        String cluster = UUID.randomUUID().toString();
+        String brokerName = UUID.randomUUID().toString();
+        String selfId = "n0";
+        String peers = String.format("n0-localhost:%d", nextPort());
+        BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName);
+        MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId);
+        BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig);
+        Thread.sleep(3000);
+
+        Assert.assertEquals(BrokerRole.SYNC_MASTER, storeConfig.getBrokerRole());
+
+
+        String topic = UUID.randomUUID().toString();
+        String consumerGroup = UUID.randomUUID().toString();
+        IntegrationTestBase.initTopic(topic, BaseConf.nsAddr, cluster, 1);
+        DefaultMQProducer producer = ProducerFactory.getRMQProducer(BaseConf.nsAddr);
+        DefaultMQPullConsumer consumer = ConsumerFactory.getRMQPullConsumer(BaseConf.nsAddr, consumerGroup);
+
+        for (int i = 0; i < 10; i++) {
+            Message message = new Message();
+            message.setTopic(topic);
+            message.setBody(("Hello" + i).getBytes());
+            SendResult sendResult = producer.send(message);
+            Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus());
+            Assert.assertEquals(0, sendResult.getMessageQueue().getQueueId());
+            Assert.assertEquals(brokerName, sendResult.getMessageQueue().getBrokerName());
+            Assert.assertEquals(i, sendResult.getQueueOffset());
+            Assert.assertNotNull(sendResult.getMsgId());
+            Assert.assertNotNull(sendResult.getOffsetMsgId());
+        }
+
+        Thread.sleep(500);
+        Assert.assertEquals(0, brokerController.getMessageStore().getMinOffsetInQueue(topic, 0));
+        Assert.assertEquals(10, brokerController.getMessageStore().getMaxOffsetInQueue(topic, 0));
+
+        MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0);
+        PullResult pullResult= consumer.pull(messageQueue, "*", 0, 32);
+        Assert.assertEquals(PullStatus.FOUND, pullResult.getPullStatus());
+        Assert.assertEquals(10, pullResult.getMsgFoundList().size());
+
+        for (int i = 0; i < 10; i++) {
+            MessageExt messageExt = pullResult.getMsgFoundList().get(i);
+            Assert.assertEquals(i, messageExt.getQueueOffset());
+            Assert.assertArrayEquals(("Hello" + i).getBytes(), messageExt.getBody());
+        }
+
+        producer.shutdown();
+        consumer.shutdown();
+        brokerController.shutdown();
+    }
+}

test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java

@@ -29,10 +29,15 @@ import org.apache.rocketmq.test.util.TestUtils;
 import org.apache.rocketmq.test.util.VerifyUtils;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static com.google.common.truth.Truth.assertThat;
 
+/**
+ * Currently, dose not support the ordered broadcast message
+ */
+@Ignore
 public class OrderMsgBroadCastIT extends BaseBroadCastIT {
     private static Logger logger = Logger.getLogger(OrderMsgBroadCastIT.class);
     private RMQNormalProducer producer = null;

test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java

@@ -51,7 +51,8 @@ public class NormalMsgDelayIT extends DelayConf {
     }
 
     @Test
-    public void testDelayLevell() {
+    public void testDelayLevel1() throws Exception {
+        Thread.sleep(3000);
         int delayLevel = 1;
         List<Object> delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize);
         producer.send(delayMsgs);