From 515bc3538d71cbe3f0fd293f420644e907b41d10 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 5 Nov 2018 20:40:06 +0800 Subject: [PATCH 001/122] Add dleger commitlog with tests --- .../common/message/MessageDecoder.java | 1 + pom.xml | 4 +- store/pom.xml | 15 + .../org/apache/rocketmq/store/CommitLog.java | 16 +- .../apache/rocketmq/store/ConsumeQueue.java | 2 +- .../rocketmq/store/DefaultMessageStore.java | 9 +- .../rocketmq/store/DispatchRequest.java | 16 +- .../store/SelectMappedBufferResult.java | 4 +- .../store/config/MessageStoreConfig.java | 36 + .../store/dleger/DLegerCommitLog.java | 629 ++++++++++++++++++ .../DLegerSelectMappedBufferResult.java | 36 + .../apache/rocketmq/store/StoreTestBase.java | 69 ++ .../store/dleger/DLegerCommitlogTest.java | 93 +++ 13 files changed, 914 insertions(+), 16 deletions(-) create mode 100644 store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java create mode 100644 store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java create mode 100644 store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java create mode 100644 store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index 1eb9f4d994..af0b638d3d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -41,6 +41,7 @@ public class MessageDecoder { public final static int MESSAGE_MAGIC_CODE = -626843481; public static final char NAME_VALUE_SEPARATOR = 1; public static final char PROPERTY_SEPARATOR = 2; + public static final int PHY_POS_POSITION = 4 + 4 + 4 + 4 + 4 + 8; public static final int BODY_SIZE_POSITION = 4 // 1 TOTALSIZE + 4 // 2 MAGICCODE + 4 // 3 BODYCRC diff --git a/pom.xml b/pom.xml index 0a8fef8758..148cc99b29 100644 --- a/pom.xml +++ b/pom.xml @@ -100,8 +100,8 @@ false true - 1.7 - 1.7 + 1.8 + 1.8 jacoco ${project.basedir}/../test/target/jacoco-it.exec diff --git a/store/pom.xml b/store/pom.xml index 59be3739de..7de04fcae5 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -28,6 +28,21 @@ rocketmq-store ${project.version} + + org.apache.rocketmq + rocketmq-dleger + 0.1-SNAPSHOT + + + org.apache.rocketmq + rocketmq-remoting + + + org.slf4j + slf4j-log4j12 + + + ${project.groupId} rocketmq-common diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 03b1151643..a1c3299050 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -45,11 +45,11 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService; public class CommitLog { // Message's MAGIC CODE daa320a7 public final static int MESSAGE_MAGIC_CODE = -626843481; - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); // End of file empty MAGIC CODE cbd43194 - private final static int BLANK_MAGIC_CODE = -875286124; + protected final static int BLANK_MAGIC_CODE = -875286124; private final MappedFileQueue mappedFileQueue; - private final DefaultMessageStore defaultMessageStore; + protected final DefaultMessageStore defaultMessageStore; private final FlushCommitLogService flushCommitLogService; //If TransientStorePool enabled, we must flush message to FileChannel at fixed periods @@ -57,11 +57,11 @@ public class CommitLog { private final AppendMessageCallback appendMessageCallback; private final ThreadLocal batchEncoderThreadLocal; - private HashMap topicQueueTable = new HashMap(1024); - private volatile long confirmOffset = -1L; + protected HashMap topicQueueTable = new HashMap(1024); + protected volatile long confirmOffset = -1L; - private volatile long beginTimeInLock = 0; - private final PutMessageLock putMessageLock; + protected volatile long beginTimeInLock = 0; + protected final PutMessageLock putMessageLock; public CommitLog(final DefaultMessageStore defaultMessageStore) { this.mappedFileQueue = new MappedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(), @@ -366,7 +366,7 @@ public class CommitLog { return new DispatchRequest(-1, false /* success */); } - private static int calMsgLength(int bodyLength, int topicLength, int propertiesLength) { + protected static int calMsgLength(int bodyLength, int topicLength, int propertiesLength) { final int msgLen = 4 //TOTALSIZE + 4 //MAGICCODE + 4 //BODYCRC diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index 08c7f99906..f5a1250d47 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -348,7 +348,7 @@ public class ConsumeQueue { long tagsCode = result.getByteBuffer().getLong(); if (offsetPy >= phyMinOffset) { - this.minLogicOffset = result.getMappedFile().getFileFromOffset() + i; + this.minLogicOffset = mappedFile.getFileFromOffset() + i; log.info("Compute logical min offset: {}, topic: {}, queueId: {}", this.getMinOffsetInQueue(), this.topic, this.queueId); // This maybe not take effect, when not every consume queue has extend file. diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index e0aef4f374..76be492ba4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -52,6 +52,7 @@ import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.dleger.DLegerCommitLog; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.index.IndexService; import org.apache.rocketmq.store.index.QueryOffsetResult; @@ -119,7 +120,11 @@ public class DefaultMessageStore implements MessageStore { this.messageStoreConfig = messageStoreConfig; this.brokerStatsManager = brokerStatsManager; this.allocateMappedFileService = new AllocateMappedFileService(this); - this.commitLog = new CommitLog(this); + if (messageStoreConfig.isEnableDLegerCommitLog()) { + this.commitLog = new DLegerCommitLog(this); + } else { + this.commitLog = new CommitLog(this); + } this.consumeQueueTable = new ConcurrentHashMap<>(32); this.flushConsumeQueueService = new FlushConsumeQueueService(); @@ -1763,7 +1768,7 @@ public class DefaultMessageStore implements MessageStore { for (int readSize = 0; readSize < result.getSize() && doNext; ) { DispatchRequest dispatchRequest = DefaultMessageStore.this.commitLog.checkMessageAndReturnSize(result.getByteBuffer(), false, false); - int size = dispatchRequest.getMsgSize(); + int size = dispatchRequest.getBufferSize() == -1 ? dispatchRequest.getMsgSize() : dispatchRequest.getBufferSize(); if (dispatchRequest.isSuccess()) { if (size > 0) { diff --git a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java index 819bb948c7..89d47ced5b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java +++ b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java @@ -22,7 +22,7 @@ public class DispatchRequest { private final String topic; private final int queueId; private final long commitLogOffset; - private final int msgSize; + private int msgSize; private final long tagsCode; private final long storeTimestamp; private final long consumeQueueOffset; @@ -35,6 +35,8 @@ public class DispatchRequest { private final Map propertiesMap; private byte[] bitMap; + private int bufferSize = -1;//the buffer size maybe larger than the msg size if the message is wrapped by something + public DispatchRequest( final String topic, final int queueId, @@ -156,4 +158,16 @@ public class DispatchRequest { public void setBitMap(byte[] bitMap) { this.bitMap = bitMap; } + + public void setMsgSize(int msgSize) { + this.msgSize = msgSize; + } + + public int getBufferSize() { + return bufferSize; + } + + public void setBufferSize(int bufferSize) { + this.bufferSize = bufferSize; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java index 7a17114c8f..1f826fedd8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java +++ b/store/src/main/java/org/apache/rocketmq/store/SelectMappedBufferResult.java @@ -48,9 +48,9 @@ public class SelectMappedBufferResult { this.byteBuffer.limit(this.size); } - public MappedFile getMappedFile() { + /* public MappedFile getMappedFile() { return mappedFile; - } + }*/ // @Override // protected void finalize() { diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 02aa84a3e6..63c7c313fc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -143,6 +143,11 @@ public class MessageStoreConfig { private int transientStorePoolSize = 5; private boolean fastFailIfNoBufferInStorePool = false; + private boolean enableDLegerCommitLog; + private String dLegerGroup; + private String dLegerPeers; + private String dLegerSelfId; + public boolean isDebugLockEnable() { return debugLockEnable; } @@ -666,4 +671,35 @@ public class MessageStoreConfig { this.commitCommitLogThoroughInterval = commitCommitLogThoroughInterval; } + public String getdLegerGroup() { + return dLegerGroup; + } + + public void setdLegerGroup(String dLegerGroup) { + this.dLegerGroup = dLegerGroup; + } + + public String getdLegerPeers() { + return dLegerPeers; + } + + public void setdLegerPeers(String dLegerPeers) { + this.dLegerPeers = dLegerPeers; + } + + public String getdLegerSelfId() { + return dLegerSelfId; + } + + public void setdLegerSelfId(String dLegerSelfId) { + this.dLegerSelfId = dLegerSelfId; + } + + public boolean isEnableDLegerCommitLog() { + return enableDLegerCommitLog; + } + + public void setEnableDLegerCommitLog(boolean enableDLegerCommitLog) { + this.enableDLegerCommitLog = enableDLegerCommitLog; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java new file mode 100644 index 0000000000..272a1868ba --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -0,0 +1,629 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.dleger; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBatch; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.dleger.DLegerConfig; +import org.apache.rocketmq.dleger.DLegerServer; +import org.apache.rocketmq.dleger.entry.DLegerEntry; +import org.apache.rocketmq.dleger.protocol.AppendEntryRequest; +import org.apache.rocketmq.dleger.protocol.AppendEntryResponse; +import org.apache.rocketmq.dleger.protocol.DLegerResponseCode; +import org.apache.rocketmq.dleger.store.file.DLegerMmapFileStore; +import org.apache.rocketmq.dleger.store.file.MmapFile; +import org.apache.rocketmq.dleger.store.file.MmapFileList; +import org.apache.rocketmq.dleger.store.file.SelectMmapBufferResult; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.CommitLog; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.StoreStatsService; +import org.apache.rocketmq.store.schedule.ScheduleMessageService; + +/** + * Store all metadata downtime for recovery, data protection reliability + */ +public class DLegerCommitLog extends CommitLog { + private final DLegerServer dLegerServer; + private final DLegerMmapFileStore dLegerFileStore; + private final MmapFileList dLegerFileList; + + + + private final MessageSerializer messageSerializer; + + private volatile long beginTimeInLock = 0; + + public DLegerCommitLog(final DefaultMessageStore defaultMessageStore) { + super(defaultMessageStore); + DLegerConfig dLegerConfig = new DLegerConfig(); + dLegerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); + dLegerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); + dLegerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); + dLegerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); + dLegerServer = new DLegerServer(dLegerConfig); + dLegerFileStore = (DLegerMmapFileStore) dLegerServer.getdLegerStore(); + DLegerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> { + assert bodyOffset == DLegerEntry.BODY_OFFSET; + buffer.position(buffer.position() + bodyOffset + MessageDecoder.PHY_POS_POSITION); + buffer.putLong(entry.getPos() + bodyOffset); + }; + dLegerFileStore.addAppendHook(appendHook); + dLegerFileList = dLegerFileStore.getDataFileList(); + this.messageSerializer = new MessageSerializer(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize()); + + } + + public boolean load() { + /*boolean result = this.mappedFileQueue.load(); + log.info("load commit log " + (result ? "OK" : "Failed")); + return result;*/ + return true; + } + + public void start() { + dLegerServer.startup(); + /* this.flushCommitLogService.start(); + + if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + this.commitLogService.start(); + }*/ + } + + public void shutdown() { + /* if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { + this.commitLogService.shutdown(); + } + this.flushCommitLogService.shutdown();*/ + dLegerServer.shutdown(); + } + + public long flush() { + dLegerFileStore.flush(); + return dLegerFileList.getFlushedWhere(); + } + + public long getMaxOffset() { + return this.dLegerFileList.getMaxWrotePosition(); + } + + public long remainHowManyDataToCommit() { + return dLegerFileList.remainHowManyDataToCommit(); + } + + public long remainHowManyDataToFlush() { + return dLegerFileList.remainHowManyDataToFlush(); + } + + public int deleteExpiredFile( + final long expiredTime, + final int deleteFilesInterval, + final long intervalForcibly, + final boolean cleanImmediately + ) { + return 0; + } + + /** + * Read CommitLog data, use data replication + */ + public SelectMappedBufferResult getData(final long offset) { + return this.getData(offset, offset == 0); + } + + private static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { + + private SelectMmapBufferResult sbr; + public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { + super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); + this.sbr = sbr; + } + + public synchronized void release() { + super.release(); + sbr.release(); + } + + } + + public SelectMappedBufferResult convertSbr(SelectMmapBufferResult sbr) { + if (sbr == null) { + return null; + } else { + return new DLegerSelectMappedBufferResult(sbr); + } + + } + + + public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { + int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound); + if (mappedFile != null) { + int pos = (int) (offset % mappedFileSize); + SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos); + return convertSbr(sbr); + } + + return null; + } + + /** + * When the normal exit, data recovery, all memory data have been flush + */ + public void recoverNormally() { + + } + public void recoverAbnormally() { + + } + + public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC) { + return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true); + } + + private void doNothingForDeadCode(final Object obj) { + if (obj != null) { + log.debug(String.valueOf(obj.hashCode())); + } + } + + /** + * check the message and returns the message size + * + * @return 0 Come the end of the file // >0 Normal messages // -1 Message checksum failure + */ + public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC, + final boolean readBody) { + try { + int bodyOffset = DLegerEntry.BODY_OFFSET; + byteBuffer.position(byteBuffer.position() + bodyOffset); + DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); + if (dispatchRequest.isSuccess()) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } else if (dispatchRequest.getMsgSize() > 0) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } + return dispatchRequest; + } catch (Exception e) { + } + + return new DispatchRequest(-1, false /* success */); + } + + public long getConfirmOffset() { + return this.confirmOffset; + } + + public void setConfirmOffset(long phyOffset) { + this.confirmOffset = phyOffset; + } + + + + + private void notifyMessageArriving() { + + } + + public boolean resetOffset(long offset) { + //return this.mappedFileQueue.resetOffset(offset); + return false; + } + + public long getBeginTimeInLock() { + return beginTimeInLock; + } + + public PutMessageResult putMessage(final MessageExtBrokerInner msg) { + // Set the storage time + msg.setStoreTimestamp(System.currentTimeMillis()); + // Set the message body BODY CRC (consider the most appropriate setting + // on the client) + msg.setBodyCRC(UtilAll.crc32(msg.getBody())); + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + String topic = msg.getTopic(); + int queueId = msg.getQueueId(); + + //should be consistent with the old version + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE + || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { + // Delay Delivery + if (msg.getDelayTimeLevel() > 0) { + if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()) { + msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel()); + } + + topic = ScheduleMessageService.SCHEDULE_TOPIC; + queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); + + // Backup real topic, queueId + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, String.valueOf(msg.getQueueId())); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + + msg.setTopic(topic); + msg.setQueueId(queueId); + } + } + + // Back to Results + AppendMessageResult appendResult = null; + PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; + CompletableFuture dlegerFuture = null; + EncodeResult encodeResult = null; + long eclipseTimeInLock = 0L; + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + long queueOffset = -1; + try { + beginTimeInLock = this.defaultMessageStore.getSystemClock().now(); + //TO DO use buffer + encodeResult = this.messageSerializer.serialize(msg); + queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + appendResult = new AppendMessageResult(encodeResult.status); + switch (encodeResult.status) { + case PROPERTIES_SIZE_EXCEEDED: + case MESSAGE_SIZE_EXCEEDED: + putMessageStatus = PutMessageStatus.MESSAGE_ILLEGAL; + break; + } + } else { + AppendEntryRequest request = new AppendEntryRequest(); + request.setRemoteId(dLegerServer.getMemberState().getSelfId()); + request.setBody(encodeResult.data); + dlegerFuture = dLegerServer.handleAppend(request); + if (dlegerFuture.isDone() && dlegerFuture.get().getCode() != DLegerResponseCode.SUCCESS.getCode()) { + //TO DO make sure the local store is ok + appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } else { + switch (tranType) { + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + // The next update ConsumeQueue information + DLegerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); + break; + default: + break; + } + } + } + eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInLock; + } catch (Exception e) { + log.error("Put message error", e); + appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } finally { + putMessageLock.unlock(); + } + + if (eclipseTimeInLock > 500) { + log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, appendResult); + } + + + if (dlegerFuture != null) { + try { + AppendEntryResponse appendEntryResponse = dlegerFuture.get(3, TimeUnit.SECONDS); + switch (DLegerResponseCode.valueOf(appendEntryResponse.getCode())) { + case SUCCESS: + putMessageStatus = PutMessageStatus.PUT_OK; + long wroteOffset = appendEntryResponse.getPos() + DLegerEntry.BODY_OFFSET; + ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, 0); + break; + case INCONSISTENT_LEADER: + case NOT_LEADER: + case NOT_READY: + case DISK_FULL: + putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; + break; + case LEADER_PENDING_FULL: + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; + } + } catch (Exception ignored) { + putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; + appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } + } + + PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); + if (putMessageStatus == PutMessageStatus.PUT_OK) { + // Statistics + storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet(); + storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(appendResult.getWroteBytes()); + } + return putMessageResult; + } + + public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) { + + } + + public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) { + + } + + public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + + /** + * According to receive certain message or offset storage time if an error occurs, it returns -1 + */ + public long pickupStoreTimestamp(final long offset, final int size) { + if (offset >= this.getMinOffset()) { + SelectMappedBufferResult result = this.getMessage(offset, size); + if (null != result) { + try { + return result.getByteBuffer().getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); + } finally { + result.release(); + } + } + } + + return -1; + } + + public long getMinOffset() { + return dLegerFileList.getMinOffset(); + } + + public SelectMappedBufferResult getMessage(final long offset, final int size) { + int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, offset == 0); + if (mappedFile != null) { + int pos = (int) (offset % mappedFileSize); + return convertSbr(mappedFile.selectMappedBuffer(pos, size)); + } + return null; + } + + public long rollNextFile(final long offset) { + int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + return offset + mappedFileSize - offset % mappedFileSize; + } + + public HashMap getTopicQueueTable() { + return topicQueueTable; + } + + public void setTopicQueueTable(HashMap topicQueueTable) { + this.topicQueueTable = topicQueueTable; + } + + public void destroy() { + //TO DO + } + + public boolean appendData(long startOffset, byte[] data) { + //TO DO + return false; + } + + public boolean retryDeleteFirstFile(final long intervalForcibly) { + //TO DO + return false; + } + + public void removeQueueFromTopicQueueTable(final String topic, final int queueId) { + String key = topic + "-" + queueId; + synchronized (this) { + this.topicQueueTable.remove(key); + } + + log.info("removeQueueFromTopicQueueTable OK Topic: {} QueueId: {}", topic, queueId); + } + + public void checkSelf() { + dLegerFileList.checkSelf(); + } + + public long lockTimeMills() { + long diff = 0; + long begin = this.beginTimeInLock; + if (begin > 0) { + diff = this.defaultMessageStore.now() - begin; + } + + if (diff < 0) { + diff = 0; + } + + return diff; + } + + class EncodeResult { + private String queueOffsetKey; + private byte[] data; + private AppendMessageStatus status; + public EncodeResult(AppendMessageStatus status, byte[] data, String queueOffsetKey) { + this.data = data; + this.status = status; + this.queueOffsetKey = queueOffsetKey; + } + } + + class MessageSerializer { + // File at the end of the minimum fixed length empty + private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; + private final ByteBuffer msgIdMemory; + // Store the message content + private final ByteBuffer msgStoreItemMemory; + // The maximum length of the message + private final int maxMessageSize; + // Build Message Key + private final StringBuilder keyBuilder = new StringBuilder(); + + private final StringBuilder msgIdBuilder = new StringBuilder(); + + private final ByteBuffer hostHolder = ByteBuffer.allocate(8); + + MessageSerializer(final int size) { + this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); + this.maxMessageSize = size; + } + + public ByteBuffer getMsgStoreItemMemory() { + return msgStoreItemMemory; + } + + public EncodeResult serialize(final MessageExtBrokerInner msgInner) { + // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
+ + // PHY OFFSET + long wroteOffset = 0; + + this.resetByteBuffer(hostHolder, 8); + // Record ConsumeQueue information + keyBuilder.setLength(0); + keyBuilder.append(msgInner.getTopic()); + keyBuilder.append('-'); + keyBuilder.append(msgInner.getQueueId()); + String key = keyBuilder.toString(); + + Long queueOffset = DLegerCommitLog.this.topicQueueTable.get(key); + if (null == queueOffset) { + queueOffset = 0L; + DLegerCommitLog.this.topicQueueTable.put(key, queueOffset); + } + + // Transaction messages that require special handling + final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag()); + switch (tranType) { + // Prepared and Rollback message is not consumed, will not enter the + // consumer queuec + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + queueOffset = 0L; + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + default: + break; + } + + /** + * Serialize message + */ + final byte[] propertiesData = + msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(MessageDecoder.CHARSET_UTF8); + + final int propertiesLength = propertiesData == null ? 0 : propertiesData.length; + + if (propertiesLength > Short.MAX_VALUE) { + log.warn("putMessage message properties length too long. length={}", propertiesData.length); + return new EncodeResult(AppendMessageStatus.PROPERTIES_SIZE_EXCEEDED, null, key); + } + + final byte[] topicData = msgInner.getTopic().getBytes(MessageDecoder.CHARSET_UTF8); + final int topicLength = topicData.length; + + final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length; + + final int msgLen = calMsgLength(bodyLength, topicLength, propertiesLength); + + // Exceeds the maximum message + if (msgLen > this.maxMessageSize) { + DLegerCommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength + + ", maxMessageSize: " + this.maxMessageSize); + return new EncodeResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED, null, key); + } + // Initialization of storage space + this.resetByteBuffer(msgStoreItemMemory, msgLen); + // 1 TOTALSIZE + this.msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + this.msgStoreItemMemory.putInt(DLegerCommitLog.MESSAGE_MAGIC_CODE); + // 3 BODYCRC + this.msgStoreItemMemory.putInt(msgInner.getBodyCRC()); + // 4 QUEUEID + this.msgStoreItemMemory.putInt(msgInner.getQueueId()); + // 5 FLAG + this.msgStoreItemMemory.putInt(msgInner.getFlag()); + // 6 QUEUEOFFSET + this.msgStoreItemMemory.putLong(queueOffset); + // 7 PHYSICALOFFSET + this.msgStoreItemMemory.putLong(wroteOffset); + // 8 SYSFLAG + this.msgStoreItemMemory.putInt(msgInner.getSysFlag()); + // 9 BORNTIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + // 10 BORNHOST + this.resetByteBuffer(hostHolder, 8); + this.msgStoreItemMemory.put(msgInner.getBornHostBytes(hostHolder)); + // 11 STORETIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + this.resetByteBuffer(hostHolder, 8); + this.msgStoreItemMemory.put(msgInner.getStoreHostBytes(hostHolder)); + //this.msgBatchMemory.put(msgInner.getStoreHostBytes()); + // 13 RECONSUMETIMES + this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + // 14 Prepared Transaction Offset + this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + // 15 BODY + this.msgStoreItemMemory.putInt(bodyLength); + if (bodyLength > 0) + this.msgStoreItemMemory.put(msgInner.getBody()); + // 16 TOPIC + this.msgStoreItemMemory.put((byte) topicLength); + this.msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + this.msgStoreItemMemory.putShort((short) propertiesLength); + if (propertiesLength > 0) + this.msgStoreItemMemory.put(propertiesData); + + final long beginTimeMills = DLegerCommitLog.this.defaultMessageStore.now(); + + byte[] data = new byte[msgLen]; + this.msgStoreItemMemory.clear(); + this.msgStoreItemMemory.get(data); + return new EncodeResult(AppendMessageStatus.PUT_OK, data, key); + } + + private void resetByteBuffer(final ByteBuffer byteBuffer, final int limit) { + byteBuffer.flip(); + byteBuffer.limit(limit); + } + + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java new file mode 100644 index 0000000000..2d21af4a57 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.dleger; + +import org.apache.rocketmq.dleger.store.file.SelectMmapBufferResult; +import org.apache.rocketmq.store.SelectMappedBufferResult; + +public class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { + + private SelectMmapBufferResult sbr; + public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { + super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); + this.sbr = sbr; + } + + @Override + public synchronized void release() { + super.release(); + sbr.release(); + } +} diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java new file mode 100644 index 0000000000..636386754e --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -0,0 +1,69 @@ +package org.apache.rocketmq.store; + +import java.io.File; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.common.UtilAll; +import org.junit.After; + +public class StoreTestBase { + + private int QUEUE_TOTAL = 100; + private AtomicInteger QueueId = new AtomicInteger(0); + private SocketAddress BornHost = new InetSocketAddress("127.0.0.1", 8123); + private SocketAddress StoreHost = BornHost; + private byte[] MessageBody = new byte[1024]; + + protected Set baseDirs = new HashSet<>(); + + private AtomicInteger port = new AtomicInteger(30000); + + public int nextPort() { + return port.incrementAndGet(); + } + + protected MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("StoreTest"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(0); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + return msg; + } + + public static String createBaseDir() { + String baseDir = System.getProperty("user.home") + File.separator + "unitteststore" + File.separator + UUID.randomUUID(); + final File file = new File(baseDir); + if (file.exists()) { + System.exit(1); + } + return baseDir; + } + + public static void deleteFile(String fileName) { + deleteFile(new File(fileName)); + } + + public static void deleteFile(File file) { + UtilAll.deleteFile(file); + } + + @After + public void clear() { + for (String baseDir : baseDirs) { + deleteFile(baseDir); + } + } + +} diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java new file mode 100644 index 0000000000..8ad58c8f40 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -0,0 +1,93 @@ +package org.apache.rocketmq.store.dleger; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.GetMessageResult; +import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MessageArrivingListener; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.StoreTestBase; +import org.apache.rocketmq.store.config.FlushDiskType; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.Assert; +import org.junit.Test; + +public class DLegerCommitlogTest extends StoreTestBase { + + private DefaultMessageStore createMessageStore(String base) throws Exception { + baseDirs.add(base); + MessageStoreConfig storeConfig = new MessageStoreConfig(); + storeConfig.setMapedFileSizeCommitLog(1024 * 100); + storeConfig.setMapedFileSizeConsumeQueue(1024); + storeConfig.setMaxHashSlotNum(100); + storeConfig.setMaxIndexNum(100 * 10); + storeConfig.setStorePathRootDir(base); + storeConfig.setStorePathCommitLog(base + File.separator + "commitlog"); + storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH); + + storeConfig.setEnableDLegerCommitLog(true); + storeConfig.setdLegerGroup(UUID.randomUUID().toString()); + storeConfig.setdLegerPeers(String.format("n0-localhost:%d", nextPort())); + storeConfig.setdLegerSelfId("n0"); + DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLegerCommitlogTest"), new MessageArrivingListener() { + + @Override + public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, + byte[] filterBitMap, Map properties) { + + } + }, new BrokerConfig()); + defaultMessageStore.load(); + defaultMessageStore.start(); + return defaultMessageStore; + } + + @Test + public void testPutAndGetMessage() throws Exception { + String base = createBaseDir(); + DefaultMessageStore messageStore = createMessageStore(base); + Thread.sleep(1000); + String topic = UUID.randomUUID().toString(); + + List results = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + MessageExtBrokerInner msgInner = buildMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(0); + PutMessageResult putMessageResult = messageStore.putMessage(msgInner); + results.add(putMessageResult); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + Thread.sleep(100); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(10, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, 0, 32, null); + Assert.assertEquals(GetMessageStatus.FOUND, getMessageResult.getStatus()); + + Assert.assertEquals(10, getMessageResult.getMessageBufferList().size()); + Assert.assertEquals(10, getMessageResult.getMessageMapedList().size()); + + for (int i = 0; i < results.size(); i++) { + ByteBuffer buffer = getMessageResult.getMessageBufferList().get(i); + MessageExt messageExt = MessageDecoder.decode(buffer); + Assert.assertEquals(i, messageExt.getQueueOffset()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId()); + Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); + } + } + + +} From 5f730028504cb130e225c2459668f208e6d03770 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 6 Nov 2018 15:33:57 +0800 Subject: [PATCH 002/122] Add role change handler for broker controller --- .../rocketmq/broker/BrokerController.java | 141 +++++++++++++----- .../dleger/DLegerRoleChangeHandler.java | 49 ++++++ .../broker/slave/SlaveSynchronize.java | 8 +- .../rocketmq/store/DefaultMessageStore.java | 30 +++- .../apache/rocketmq/store/MessageStore.java | 7 + .../store/dleger/DLegerCommitLog.java | 4 + .../store/dleger/DLegerCommitlogTest.java | 1 + .../test/base/IntegrationTestBase.java | 10 +- 8 files changed, 190 insertions(+), 60 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index e7ef46d0c6..d5d6a53a36 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; @@ -38,6 +39,7 @@ import org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ProducerManager; import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager; +import org.apache.rocketmq.broker.dleger.DLegerRoleChangeHandler; import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filtersrv.FilterServerManager; @@ -97,6 +99,7 @@ import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.dleger.DLegerCommitLog; import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -156,6 +159,7 @@ public class BrokerController { private TransactionalMessageCheckService transactionalMessageCheckService; private TransactionalMessageService transactionalMessageService; private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; + private Future slaveSyncFuture; public BrokerController( final BrokerConfig brokerConfig, @@ -231,6 +235,10 @@ public class BrokerController { this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); + if (messageStoreConfig.isEnableDLegerCommitLog()) { + DLegerRoleChangeHandler roleChangeHandler = new DLegerRoleChangeHandler(this, (DefaultMessageStore) messageStore); + ((DLegerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLegerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); + } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin MessageStorePluginContext context = new MessageStorePluginContext(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig); @@ -392,40 +400,6 @@ public class BrokerController { } }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); } - - if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { - if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) { - this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress()); - this.updateMasterHAServerAddrPeriodically = false; - } else { - this.updateMasterHAServerAddrPeriodically = true; - } - - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - try { - BrokerController.this.slaveSynchronize.syncAll(); - } catch (Throwable e) { - log.error("ScheduledTask syncAll slave exception", e); - } - } - }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); - } else { - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - try { - BrokerController.this.printMasterAndSlaveDiff(); - } catch (Throwable e) { - log.error("schedule printMasterAndSlaveDiff error.", e); - } - } - }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); - } - if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { // Register a listener to reload SslContext try { @@ -831,11 +805,9 @@ public class BrokerController { this.brokerFastFailure.start(); } - if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { - if (this.transactionalMessageCheckService != null) { - log.info("Start transaction service!"); - this.transactionalMessageCheckService.start(); - } + if (!messageStoreConfig.isEnableDLegerCommitLog()) { + startProcessorByHa(); + handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); } } @@ -1052,4 +1024,95 @@ public class BrokerController { public BlockingQueue getEndTransactionThreadPoolQueue() { return endTransactionThreadPoolQueue; } + + + + private void handleSlaveSynchronize(BrokerRole role) { + if (role == BrokerRole.SLAVE) { + if (null != slaveSyncFuture){ + slaveSyncFuture.cancel(false); + } + this.slaveSynchronize.setMasterAddr(null); + slaveSyncFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.slaveSynchronize.syncAll(); + } + catch (Throwable e) { + log.error("ScheduledTask SlaveSynchronize syncAll error.", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } else { + //handle the slave synchronise + if (null != slaveSyncFuture){ + slaveSyncFuture.cancel(false); + } + this.slaveSynchronize.setMasterAddr(null); + } + } + + public void changeToSlave() { + //change the role + brokerConfig.setBrokerId(1); //TO DO check + messageStoreConfig.setBrokerRole(BrokerRole.SLAVE); + + //handle the scheduled service + this.messageStore.handleScheduleMessageService(BrokerRole.SLAVE); + + //handle the transactional service + this.shutdownProcessorByHa(); + + //handle the slave synchronise + handleSlaveSynchronize(BrokerRole.SLAVE); + + try { + this.registerBrokerAll(true, true, brokerConfig.isForceRegister()); + } catch (Throwable ignored) { + + } + } + + + + public void changeToMaster(BrokerRole role) { + if (role == BrokerRole.SLAVE) { + return; + } + //handle the slave synchronise + handleSlaveSynchronize(role); + //change the role + brokerConfig.setBrokerId(0); //TO DO check + messageStoreConfig.setBrokerRole(role); + + //handle the scheduled service + this.messageStore.handleScheduleMessageService(role); + + //handle the transactional service + this.startProcessorByHa(); + + try { + this.registerBrokerAll(true, true, brokerConfig.isForceRegister()); + } catch (Throwable ignored) { + + } + } + + private void startProcessorByHa() { + if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { + if (this.transactionalMessageCheckService != null) { + this.transactionalMessageCheckService.start(); + } + } + } + + private void shutdownProcessorByHa() { + if (this.transactionalMessageCheckService != null) { + this.transactionalMessageCheckService.shutdown(true); + } + } + + + } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java new file mode 100644 index 0000000000..3d3b63a121 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -0,0 +1,49 @@ +package org.apache.rocketmq.broker.dleger; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.dleger.DLegerLeaderElector; +import org.apache.rocketmq.dleger.MemberState; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.config.BrokerRole; + +public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private BrokerController brokerController; + private DefaultMessageStore messageStore; + public DLegerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { + this.brokerController = brokerController; + this.messageStore = messageStore; + } + + @Override public void handle(long term, MemberState.Role role) { + try { + log.info("Begin handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + switch (role) { + case CANDIDATE: + if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { + brokerController.changeToSlave(); + } + break; + case FOLLOWER: + brokerController.changeToSlave(); + break; + case LEADER: + while (messageStore.dispatchBehindBytes() != 0) { + Thread.sleep(100); + } + messageStore.recoverTopicQueueTable(); + brokerController.changeToMaster(BrokerRole.SYNC_MASTER); + break; + default: + break; + } + log.info("Finish handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + } catch (Throwable t) { + log.info("Failed handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), t); + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 643a812fa0..7b5e5645a0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -54,7 +54,7 @@ public class SlaveSynchronize { private void syncTopicConfig() { String masterAddrBak = this.masterAddr; - if (masterAddrBak != null) { + if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) { try { TopicConfigSerializeWrapper topicWrapper = this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak); @@ -78,7 +78,7 @@ public class SlaveSynchronize { private void syncConsumerOffset() { String masterAddrBak = this.masterAddr; - if (masterAddrBak != null) { + if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) { try { ConsumerOffsetSerializeWrapper offsetWrapper = this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak); @@ -94,7 +94,7 @@ public class SlaveSynchronize { private void syncDelayOffset() { String masterAddrBak = this.masterAddr; - if (masterAddrBak != null) { + if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) { try { String delayOffset = this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak); @@ -118,7 +118,7 @@ public class SlaveSynchronize { private void syncSubscriptionGroupConfig() { String masterAddrBak = this.masterAddr; - if (masterAddrBak != null) { + if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) { try { SubscriptionGroupWrapper subscriptionWrapper = this.brokerController.getBrokerOuterAPI() diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 76be492ba4..cd0c46def2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -226,10 +226,6 @@ public class DefaultMessageStore implements MessageStore { this.commitLog.start(); this.storeStatsService.start(); - if (this.scheduleMessageService != null && SLAVE != messageStoreConfig.getBrokerRole()) { - this.scheduleMessageService.start(); - } - if (this.getMessageStoreConfig().isDuplicationEnable()) { this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset()); } else { @@ -237,13 +233,18 @@ public class DefaultMessageStore implements MessageStore { } this.reputMessageService.start(); - this.haService.start(); + if (!messageStoreConfig.isEnableDLegerCommitLog()) { + this.haService.start(); + this.handleScheduleMessageService(messageStoreConfig.getBrokerRole()); + } this.createTempFile(); this.addScheduleTask(); this.shutdown = false; } + + public void shutdown() { if (!this.shutdown) { this.shutdown = true; @@ -260,8 +261,9 @@ public class DefaultMessageStore implements MessageStore { if (this.scheduleMessageService != null) { this.scheduleMessageService.shutdown(); } - - this.haService.shutdown(); + if (this.haService != null) { + this.haService.shutdown(); + } this.storeStatsService.shutdown(); this.indexService.shutdown(); @@ -1325,7 +1327,7 @@ public class DefaultMessageStore implements MessageStore { return maxPhysicOffset; } - private void recoverTopicQueueTable() { + public void recoverTopicQueueTable() { HashMap table = new HashMap(1024); long minPhyOffset = this.commitLog.getMinOffset(); for (ConcurrentMap maps : this.consumeQueueTable.values()) { @@ -1387,6 +1389,18 @@ public class DefaultMessageStore implements MessageStore { return brokerStatsManager; } + @Override + public void handleScheduleMessageService(final BrokerRole brokerRole) { + if (this.scheduleMessageService != null) { + if (brokerRole == BrokerRole.SLAVE) { + this.scheduleMessageService.shutdown(); + } else { + this.scheduleMessageService.start(); + } + } + + } + public int remainTransientStoreBufferNumbs() { return this.transientStorePool.remainBufferNumbs(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 0f9b4f0ae6..5a046ff181 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -21,6 +21,7 @@ import java.util.LinkedList; import java.util.Set; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; +import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.stats.BrokerStatsManager; /** @@ -366,4 +367,10 @@ public interface MessageStore { * @return BrokerStatsManager. */ BrokerStatsManager getBrokerStatsManager(); + + /** + * handle + * @param brokerRole + */ + void handleScheduleMessageService(BrokerRole brokerRole); } diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 272a1868ba..16a2f27486 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -626,4 +626,8 @@ public class DLegerCommitLog extends CommitLog { } } + + public DLegerServer getdLegerServer() { + return dLegerServer; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java index 8ad58c8f40..67c2ed9164 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -87,6 +87,7 @@ public class DLegerCommitlogTest extends StoreTestBase { Assert.assertEquals(results.get(i).getAppendMessageResult().getMsgId(), messageExt.getMsgId()); Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); } + messageStore.destroy(); } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 9dcb2d8b75..48d9a266e0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -179,15 +179,7 @@ public class IntegrationTestBase { if (!file.exists()) { return; } - if (file.isFile()) { - file.delete(); - } else if (file.isDirectory()) { - File[] files = file.listFiles(); - for (File file1 : files) { - deleteFile(file1); - } - file.delete(); - } + UtilAll.deleteFile(file); } } From 456691bf1ef16cc45535d638c170cb90df509d7b Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 6 Nov 2018 16:34:03 +0800 Subject: [PATCH 003/122] Add IT tests for dleger commitlog --- .../dleger/DLegerRoleChangeHandler.java | 6 +- .../rocketmq/store/DefaultMessageStore.java | 7 +- .../test/factory/ConsumerFactory.java | 8 ++ .../apache/rocketmq/test/base/BaseConf.java | 2 +- .../test/base/IntegrationTestBase.java | 28 +++-- .../base/dleger/ProduceAndConsumeTest.java | 102 ++++++++++++++++++ 6 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 3d3b63a121..3fcf588b9b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -21,7 +21,7 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa @Override public void handle(long term, MemberState.Role role) { try { - log.info("Begin handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); switch (role) { case CANDIDATE: if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { @@ -41,9 +41,9 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa default: break; } - log.info("Finish handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + log.info("Finish handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); } catch (Throwable t) { - log.info("Failed handling lastRole change term={} lastRole={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), t); + log.info("Failed handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), t); } } } diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index cd0c46def2..8f5c4e80f4 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -132,8 +132,11 @@ public class DefaultMessageStore implements MessageStore { this.cleanConsumeQueueService = new CleanConsumeQueueService(); this.storeStatsService = new StoreStatsService(); this.indexService = new IndexService(this); - this.haService = new HAService(this); - + if (!messageStoreConfig.isEnableDLegerCommitLog()) { + this.haService = new HAService(this); + } else { + this.haService = null; + } this.reputMessageService = new ReputMessageService(); this.scheduleMessageService = new ScheduleMessageService(this); diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java index 0f94f3606c..c37cf1f96b 100644 --- a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java +++ b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.factory; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; @@ -60,4 +61,11 @@ public class ConsumerFactory { consumer.start(); return consumer; } + + public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception { + DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); + defaultMQPullConsumer.setNamesrvAddr(nsAddr); + defaultMQPullConsumer.start(); + return defaultMQPullConsumer; + } } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index 5027a3cce0..1f4fe7175e 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -33,7 +33,7 @@ import org.apache.rocketmq.test.util.MQAdmin; import org.apache.rocketmq.test.util.MQRandomUtils; public class BaseConf { - protected static String nsAddr; + public static String nsAddr; protected static String broker1Name; protected static String broker2Name; protected static String clusterName; diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 48d9a266e0..8890a58739 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -50,6 +50,11 @@ public class IntegrationTestBase { protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 256; protected static final int INDEX_NUM = 1000; + private static final AtomicInteger port = new AtomicInteger(50000); + + public static synchronized int nextPort() { + return port.addAndGet(5); + } protected static Random random = new Random(); static { @@ -87,7 +92,7 @@ public class IntegrationTestBase { } - private static String createBaseDir() { + public static String createBaseDir() { String baseDir = System.getProperty("user.home") + SEP + "unitteststore-" + UUID.randomUUID(); final File file = new File(baseDir); if (file.exists()) { @@ -112,7 +117,7 @@ public class IntegrationTestBase { logger.info("Name Server Start:{}", nameServerNettyServerConfig.getListenPort()); namesrvController.start(); } catch (Exception e) { - logger.info("Name Server start failed"); + logger.info("Name Server start failed", e); System.exit(1); } NAMESRV_CONTROLLERS.add(namesrvController); @@ -123,8 +128,6 @@ public class IntegrationTestBase { public static BrokerController createAndStartBroker(String nsAddr) { String baseDir = createBaseDir(); BrokerConfig brokerConfig = new BrokerConfig(); - NettyServerConfig nettyServerConfig = new NettyServerConfig(); - NettyClientConfig nettyClientConfig = new NettyClientConfig(); MessageStoreConfig storeConfig = new MessageStoreConfig(); brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.getAndIncrement()); brokerConfig.setBrokerIP1("127.0.0.1"); @@ -132,18 +135,25 @@ public class IntegrationTestBase { brokerConfig.setEnablePropertyFilter(true); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog"); - storeConfig.setHaListenPort(8000 + random.nextInt(1000)); - storeConfig.setMapedFileSizeCommitLog(COMMIT_LOG_SIZE); + storeConfig.setMapedFileSizeCommitLog(100 * 1024 * 1024); storeConfig.setMaxIndexNum(INDEX_NUM); storeConfig.setMaxHashSlotNum(INDEX_NUM * 4); - nettyServerConfig.setListenPort(10000 + random.nextInt(1000)); + return createAndStartBroker(storeConfig, brokerConfig); + + } + + public static BrokerController createAndStartBroker(MessageStoreConfig storeConfig, BrokerConfig brokerConfig) { + NettyServerConfig nettyServerConfig = new NettyServerConfig(); + NettyClientConfig nettyClientConfig = new NettyClientConfig(); + nettyServerConfig.setListenPort(nextPort()); + storeConfig.setHaListenPort(nextPort()); BrokerController brokerController = new BrokerController(brokerConfig, nettyServerConfig, nettyClientConfig, storeConfig); try { Assert.assertTrue(brokerController.initialize()); logger.info("Broker Start name:{} addr:{}", brokerConfig.getBrokerName(), brokerController.getBrokerAddr()); brokerController.start(); - } catch (Exception e) { - logger.info("Broker start failed"); + } catch (Throwable t) { + logger.error("Broker start failed, will exit", t); System.exit(1); } BROKER_CONTROLLERS.add(brokerController); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java b/test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java new file mode 100644 index 0000000000..30ddff4f24 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java @@ -0,0 +1,102 @@ +package org.apache.rocketmq.test.base.dleger; + +import java.util.UUID; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.base.IntegrationTestBase; +import org.apache.rocketmq.test.factory.ConsumerFactory; +import org.apache.rocketmq.test.factory.ProducerFactory; +import org.junit.Assert; +import org.junit.Test; + +import static org.apache.rocketmq.test.base.IntegrationTestBase.nextPort; +import static sun.util.locale.BaseLocale.SEP; + +public class ProduceAndConsumeTest { + + public BrokerConfig buildBrokerConfig(String cluster, String brokerName) { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setBrokerClusterName(cluster); + brokerConfig.setBrokerName(brokerName); + brokerConfig.setBrokerIP1("127.0.0.1"); + brokerConfig.setNamesrvAddr(BaseConf.nsAddr); + return brokerConfig; + } + + public MessageStoreConfig buildStoreConfig(String brokerName, String peers, String selfId) { + MessageStoreConfig storeConfig = new MessageStoreConfig(); + String baseDir = IntegrationTestBase.createBaseDir(); + storeConfig.setStorePathRootDir(baseDir); + storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog"); + storeConfig.setHaListenPort(nextPort()); + storeConfig.setMapedFileSizeCommitLog(10 * 1024 * 1024); + storeConfig.setEnableDLegerCommitLog(true); + storeConfig.setdLegerGroup(brokerName); + storeConfig.setdLegerSelfId(selfId); + storeConfig.setdLegerPeers(peers); + return storeConfig; + } + + @Test + public void testProduceAndConsume() throws Exception { + String cluster = UUID.randomUUID().toString(); + String brokerName = UUID.randomUUID().toString(); + String selfId = "n0"; + String peers = String.format("n0-localhost:%d", nextPort()); + BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName); + MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId); + BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig); + Thread.sleep(1000); + + Assert.assertEquals(BrokerRole.SYNC_MASTER, storeConfig.getBrokerRole()); + + + String topic = UUID.randomUUID().toString(); + String consumerGroup = UUID.randomUUID().toString(); + IntegrationTestBase.initTopic(topic, BaseConf.nsAddr, cluster, 1); + DefaultMQProducer producer = ProducerFactory.getRMQProducer(BaseConf.nsAddr); + DefaultMQPullConsumer consumer = ConsumerFactory.getRMQPullConsumer(BaseConf.nsAddr, consumerGroup); + + for (int i = 0; i < 10; i++) { + Message message = new Message(); + message.setTopic(topic); + message.setBody(("Hello" + i).getBytes()); + SendResult sendResult = producer.send(message); + Assert.assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus()); + Assert.assertEquals(0, sendResult.getMessageQueue().getQueueId()); + Assert.assertEquals(brokerName, sendResult.getMessageQueue().getBrokerName()); + Assert.assertEquals(i, sendResult.getQueueOffset()); + Assert.assertNotNull(sendResult.getMsgId()); + Assert.assertNotNull(sendResult.getOffsetMsgId()); + } + + Thread.sleep(500); + Assert.assertEquals(0, brokerController.getMessageStore().getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(10, brokerController.getMessageStore().getMaxOffsetInQueue(topic, 0)); + + MessageQueue messageQueue = new MessageQueue(topic, brokerName, 0); + PullResult pullResult= consumer.pull(messageQueue, "*", 0, 32); + Assert.assertEquals(PullStatus.FOUND, pullResult.getPullStatus()); + Assert.assertEquals(10, pullResult.getMsgFoundList().size()); + + for (int i = 0; i < 10; i++) { + MessageExt messageExt = pullResult.getMsgFoundList().get(i); + Assert.assertEquals(i, messageExt.getQueueOffset()); + Assert.assertArrayEquals(("Hello" + i).getBytes(), messageExt.getBody()); + } + + brokerController.shutdown(); + } +} From 1a2fe9f81b4b6d671520cd481ef86f5a18c43ffe Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 6 Nov 2018 17:36:40 +0800 Subject: [PATCH 004/122] Test the committed pos for dleger --- .../dleger/DLegerRoleChangeHandler.java | 6 +- .../store/dleger/DLegerCommitLog.java | 34 ++++++++--- .../store/dleger/DLegerCommitlogTest.java | 59 +++++++++++++++---- 3 files changed, 78 insertions(+), 21 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 3fcf588b9b..267f737279 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -4,6 +4,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.dleger.DLegerLeaderElector; import org.apache.rocketmq.dleger.MemberState; +import org.apache.rocketmq.dleger.utils.UtilAll; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; @@ -20,6 +21,7 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa } @Override public void handle(long term, MemberState.Role role) { + long start = System.currentTimeMillis(); try { log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); switch (role) { @@ -41,9 +43,9 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa default: break; } - log.info("Finish handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + log.info("Finish handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); } catch (Throwable t) { - log.info("Failed handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), t); + log.info("Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); } } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 16a2f27486..3b0ee87a60 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -133,12 +133,6 @@ public class DLegerCommitLog extends CommitLog { return 0; } - /** - * Read CommitLog data, use data replication - */ - public SelectMappedBufferResult getData(final long offset) { - return this.getData(offset, offset == 0); - } private static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { @@ -164,14 +158,37 @@ public class DLegerCommitLog extends CommitLog { } + public SelectMmapBufferResult truncate(SelectMmapBufferResult sbr) { + long committedPos = dLegerFileStore.getCommittedPos(); + if (sbr == null || sbr.getStartOffset() == committedPos) { + return null; + } + if (sbr.getStartOffset() + sbr.getSize() <= committedPos) { + return sbr; + } else { + sbr.setSize((int) (committedPos - sbr.getStartOffset())); + return sbr; + } + } + + /** + * Read CommitLog data, use data replication + */ + public SelectMappedBufferResult getData(final long offset) { + return this.getData(offset, offset == 0); + } + public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { + if (offset >= dLegerFileStore.getCommittedPos()) { + return null; + } int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos); - return convertSbr(sbr); + return convertSbr(truncate(sbr)); } return null; @@ -353,6 +370,9 @@ public class DLegerCommitLog extends CommitLog { case DISK_FULL: putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; break; + case WAIT_QUORUM_ACK_TIMEOUT: + putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; + break; case LEADER_PENDING_FULL: putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; break; diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java index 67c2ed9164..193bd50c40 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -4,15 +4,14 @@ import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.UUID; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.dleger.DLegerServer; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; -import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; @@ -25,7 +24,7 @@ import org.junit.Test; public class DLegerCommitlogTest extends StoreTestBase { - private DefaultMessageStore createMessageStore(String base) throws Exception { + private DefaultMessageStore createMessageStore(String base, String selfId, String peers, String leaderId) throws Exception { baseDirs.add(base); MessageStoreConfig storeConfig = new MessageStoreConfig(); storeConfig.setMapedFileSizeCommitLog(1024 * 100); @@ -38,16 +37,21 @@ public class DLegerCommitlogTest extends StoreTestBase { storeConfig.setEnableDLegerCommitLog(true); storeConfig.setdLegerGroup(UUID.randomUUID().toString()); - storeConfig.setdLegerPeers(String.format("n0-localhost:%d", nextPort())); - storeConfig.setdLegerSelfId("n0"); - DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLegerCommitlogTest"), new MessageArrivingListener() { + storeConfig.setdLegerPeers(peers); + storeConfig.setdLegerSelfId(selfId); + DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLegerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { - @Override - public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, - byte[] filterBitMap, Map properties) { - - } }, new BrokerConfig()); + if (leaderId != null) { + DLegerServer dLegerServer = ((DLegerCommitLog) defaultMessageStore.getCommitLog()).getdLegerServer(); + dLegerServer.getdLegerConfig().setEnableLeaderElector(false); + if (selfId.equals(leaderId)) { + dLegerServer.getMemberState().changeToLeader(-1); + } else { + dLegerServer.getMemberState().changeToFollower(-1, leaderId); + } + + } defaultMessageStore.load(); defaultMessageStore.start(); return defaultMessageStore; @@ -56,7 +60,8 @@ public class DLegerCommitlogTest extends StoreTestBase { @Test public void testPutAndGetMessage() throws Exception { String base = createBaseDir(); - DefaultMessageStore messageStore = createMessageStore(base); + String peers = String.format("n0-localhost:%d", nextPort()); + DefaultMessageStore messageStore = createMessageStore(base, "n0", peers, null); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); @@ -91,4 +96,34 @@ public class DLegerCommitlogTest extends StoreTestBase { } + @Test + public void testCommittedPos() throws Exception { + String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); + DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), "n0", peers, "n0"); + + String topic = UUID.randomUUID().toString(); + MessageExtBrokerInner msgInner = buildMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(0); + PutMessageResult putMessageResult = leaderStore.putMessage(msgInner); + Assert.assertEquals(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, putMessageResult.getPutMessageStatus()); + + Thread.sleep(1000); + + Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0); + Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); + + + DefaultMessageStore followerStore = createMessageStore(createBaseDir(), "n1", peers, "n0"); + Thread.sleep(2000); + + Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(1, followerStore.getMaxOffsetInQueue(topic, 0)); + + + leaderStore.destroy(); + followerStore.destroy(); + } + + } From dbc5987b4ac6460f0763b52154078ace35566f24 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 7 Nov 2018 11:45:54 +0800 Subject: [PATCH 005/122] Fix tests --- .../test/java/org/apache/rocketmq/store/StoreTestBase.java | 6 +++--- .../apache/rocketmq/store/dleger/DLegerCommitlogTest.java | 4 ++++ .../org/apache/rocketmq/test/base/IntegrationTestBase.java | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java index 636386754e..9e09d8132d 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -21,10 +21,10 @@ public class StoreTestBase { protected Set baseDirs = new HashSet<>(); - private AtomicInteger port = new AtomicInteger(30000); + private static AtomicInteger port = new AtomicInteger(30000); - public int nextPort() { - return port.incrementAndGet(); + public static synchronized int nextPort() { + return port.addAndGet(5); } protected MessageExtBrokerInner buildMessage() { diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java index 193bd50c40..2a247e4d89 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -93,6 +93,7 @@ public class DLegerCommitlogTest extends StoreTestBase { Assert.assertEquals(results.get(i).getAppendMessageResult().getWroteOffset(), messageExt.getCommitLogOffset()); } messageStore.destroy(); + messageStore.shutdown(); } @@ -123,6 +124,9 @@ public class DLegerCommitlogTest extends StoreTestBase { leaderStore.destroy(); followerStore.destroy(); + + leaderStore.shutdown(); + followerStore.shutdown(); } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 8890a58739..0ac321e4a3 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -47,7 +47,7 @@ public class IntegrationTestBase { protected static final List BROKER_CONTROLLERS = new ArrayList<>(); protected static final List NAMESRV_CONTROLLERS = new ArrayList<>(); protected static int topicCreateTime = 30 * 1000; - protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 256; + protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 100; protected static final int INDEX_NUM = 1000; private static final AtomicInteger port = new AtomicInteger(50000); @@ -135,7 +135,7 @@ public class IntegrationTestBase { brokerConfig.setEnablePropertyFilter(true); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog"); - storeConfig.setMapedFileSizeCommitLog(100 * 1024 * 1024); + storeConfig.setMapedFileSizeCommitLog(COMMIT_LOG_SIZE); storeConfig.setMaxIndexNum(INDEX_NUM); storeConfig.setMaxHashSlotNum(INDEX_NUM * 4); return createAndStartBroker(storeConfig, brokerConfig); From 29fb33391143eb3efc9df5aa5b92841c351bbf11 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Fri, 9 Nov 2018 16:05:39 +0800 Subject: [PATCH 006/122] fix --- .../java/org/apache/rocketmq/store/DefaultMessageStore.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 8f5c4e80f4..b3c1e7ba27 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -42,13 +42,13 @@ import org.apache.rocketmq.common.SystemClock; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.running.RunningStats; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; @@ -59,8 +59,6 @@ import org.apache.rocketmq.store.index.QueryOffsetResult; import org.apache.rocketmq.store.schedule.ScheduleMessageService; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import static org.apache.rocketmq.store.config.BrokerRole.SLAVE; - public class DefaultMessageStore implements MessageStore { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); From cc3f9f3b3a47182ebb5b764d64532e01e0a33512 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 14:35:16 +0800 Subject: [PATCH 007/122] Fix check style --- .gitignore | 3 ++- .../apache/rocketmq/broker/BrokerController.java | 4 ++-- .../broker/dleger/DLegerRoleChangeHandler.java | 16 ++++++++++++++++ .../rocketmq/store/dleger/DLegerCommitLog.java | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 80c6f56986..c4e43cd66f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ devenv *.versionsBackup !NOTICE-BIN !LICENSE-BIN -.DS_Store \ No newline at end of file +.DS_Store +localbin/ diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index d5d6a53a36..a6aecf807f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -1029,7 +1029,7 @@ public class BrokerController { private void handleSlaveSynchronize(BrokerRole role) { if (role == BrokerRole.SLAVE) { - if (null != slaveSyncFuture){ + if (null != slaveSyncFuture) { slaveSyncFuture.cancel(false); } this.slaveSynchronize.setMasterAddr(null); @@ -1046,7 +1046,7 @@ public class BrokerController { }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); } else { //handle the slave synchronise - if (null != slaveSyncFuture){ + if (null != slaveSyncFuture) { slaveSyncFuture.cancel(false); } this.slaveSynchronize.setMasterAddr(null); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 267f737279..9503fc8e34 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.broker.dleger; import org.apache.rocketmq.broker.BrokerController; diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 3b0ee87a60..dd84d3161f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -366,7 +366,7 @@ public class DLegerCommitLog extends CommitLog { break; case INCONSISTENT_LEADER: case NOT_LEADER: - case NOT_READY: + case LEADER_NOT_READY: case DISK_FULL: putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; break; From b3ba26686221cc62d859cf8388a287c66a7277fb Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 16:06:03 +0800 Subject: [PATCH 008/122] Fix os pache cache busy problem --- .../apache/rocketmq/broker/BrokerController.java | 14 ++++++++++---- .../rocketmq/store/dleger/DLegerCommitLog.java | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a6aecf807f..9f66ed8ec8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -783,6 +783,15 @@ public class BrokerController { this.filterServerManager.start(); } + if (messageStoreConfig.isEnableDLegerCommitLog()) { + changeToSlave(); + } else { + startProcessorByHa(); + handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); + } + + + this.registerBrokerAll(true, false, true); this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @@ -805,10 +814,7 @@ public class BrokerController { this.brokerFastFailure.start(); } - if (!messageStoreConfig.isEnableDLegerCommitLog()) { - startProcessorByHa(); - handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); - } + } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index dd84d3161f..cc24679956 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -345,6 +345,7 @@ public class DLegerCommitLog extends CommitLog { log.error("Put message error", e); appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); } finally { + beginTimeInLock = 0; putMessageLock.unlock(); } @@ -352,7 +353,6 @@ public class DLegerCommitLog extends CommitLog { log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, appendResult); } - if (dlegerFuture != null) { try { AppendEntryResponse appendEntryResponse = dlegerFuture.get(3, TimeUnit.SECONDS); From d2cafcf6bf1aafe17a33f7f9d7e9fb150f2df81d Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 16:27:19 +0800 Subject: [PATCH 009/122] Fix locktime and group --- .../org/apache/rocketmq/store/CommitLog.java | 2 +- .../rocketmq/store/dleger/DLegerCommitLog.java | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index a1c3299050..d10e30ba7e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -60,7 +60,7 @@ public class CommitLog { protected HashMap topicQueueTable = new HashMap(1024); protected volatile long confirmOffset = -1L; - protected volatile long beginTimeInLock = 0; + private volatile long beginTimeInLock = 0; protected final PutMessageLock putMessageLock; public CommitLog(final DefaultMessageStore defaultMessageStore) { diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index cc24679956..7bef4f19a6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -54,6 +54,7 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService; */ public class DLegerCommitLog extends CommitLog { private final DLegerServer dLegerServer; + private final DLegerConfig dLegerConfig; private final DLegerMmapFileStore dLegerFileStore; private final MmapFileList dLegerFileList; @@ -61,11 +62,11 @@ public class DLegerCommitLog extends CommitLog { private final MessageSerializer messageSerializer; - private volatile long beginTimeInLock = 0; + private volatile long beginTimeInDlegerLock = 0; public DLegerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); - DLegerConfig dLegerConfig = new DLegerConfig(); + dLegerConfig = new DLegerConfig(); dLegerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); dLegerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); dLegerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); @@ -257,8 +258,9 @@ public class DLegerCommitLog extends CommitLog { return false; } + @Override public long getBeginTimeInLock() { - return beginTimeInLock; + return beginTimeInDlegerLock; } public PutMessageResult putMessage(final MessageExtBrokerInner msg) { @@ -305,7 +307,7 @@ public class DLegerCommitLog extends CommitLog { putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long queueOffset = -1; try { - beginTimeInLock = this.defaultMessageStore.getSystemClock().now(); + beginTimeInDlegerLock = this.defaultMessageStore.getSystemClock().now(); //TO DO use buffer encodeResult = this.messageSerializer.serialize(msg); queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); @@ -319,6 +321,7 @@ public class DLegerCommitLog extends CommitLog { } } else { AppendEntryRequest request = new AppendEntryRequest(); + request.setGroup(dLegerConfig.getGroup()); request.setRemoteId(dLegerServer.getMemberState().getSelfId()); request.setBody(encodeResult.data); dlegerFuture = dLegerServer.handleAppend(request); @@ -340,12 +343,12 @@ public class DLegerCommitLog extends CommitLog { } } } - eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInLock; + eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDlegerLock; } catch (Exception e) { log.error("Put message error", e); appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); } finally { - beginTimeInLock = 0; + beginTimeInDlegerLock = 0; putMessageLock.unlock(); } @@ -476,9 +479,10 @@ public class DLegerCommitLog extends CommitLog { dLegerFileList.checkSelf(); } + @Override public long lockTimeMills() { long diff = 0; - long begin = this.beginTimeInLock; + long begin = this.beginTimeInDlegerLock; if (begin > 0) { diff = this.defaultMessageStore.now() - begin; } From aed6f0ed2aac28230543e1f584cc27991f547a9d Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 16:35:52 +0800 Subject: [PATCH 010/122] Fix tests for dleger commitlog, add group check --- .../rocketmq/store/dleger/DLegerCommitlogTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java index 2a247e4d89..be606654d7 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -24,7 +24,7 @@ import org.junit.Test; public class DLegerCommitlogTest extends StoreTestBase { - private DefaultMessageStore createMessageStore(String base, String selfId, String peers, String leaderId) throws Exception { + private DefaultMessageStore createMessageStore(String base, String group, String selfId, String peers, String leaderId) throws Exception { baseDirs.add(base); MessageStoreConfig storeConfig = new MessageStoreConfig(); storeConfig.setMapedFileSizeCommitLog(1024 * 100); @@ -36,7 +36,7 @@ public class DLegerCommitlogTest extends StoreTestBase { storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH); storeConfig.setEnableDLegerCommitLog(true); - storeConfig.setdLegerGroup(UUID.randomUUID().toString()); + storeConfig.setdLegerGroup(group); storeConfig.setdLegerPeers(peers); storeConfig.setdLegerSelfId(selfId); DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLegerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { @@ -61,7 +61,8 @@ public class DLegerCommitlogTest extends StoreTestBase { public void testPutAndGetMessage() throws Exception { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); - DefaultMessageStore messageStore = createMessageStore(base, "n0", peers, null); + String group = UUID.randomUUID().toString(); + DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); @@ -100,7 +101,8 @@ public class DLegerCommitlogTest extends StoreTestBase { @Test public void testCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); - DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), "n0", peers, "n0"); + String group = UUID.randomUUID().toString(); + DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), group,"n0", peers, "n0"); String topic = UUID.randomUUID().toString(); MessageExtBrokerInner msgInner = buildMessage(); @@ -115,7 +117,7 @@ public class DLegerCommitlogTest extends StoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createMessageStore(createBaseDir(), "n1", peers, "n0"); + DefaultMessageStore followerStore = createMessageStore(createBaseDir(), group,"n1", peers, "n0"); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); From 5020f3252633fab98ecb9a37c8dca2c368d31abb Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 17:08:06 +0800 Subject: [PATCH 011/122] Add id for dleger commitlog --- .../java/org/apache/rocketmq/broker/BrokerController.java | 6 +++--- .../rocketmq/broker/dleger/DLegerRoleChangeHandler.java | 7 +++++-- .../org/apache/rocketmq/store/dleger/DLegerCommitLog.java | 7 +++++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 9f66ed8ec8..a852600665 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -784,7 +784,7 @@ public class BrokerController { } if (messageStoreConfig.isEnableDLegerCommitLog()) { - changeToSlave(); + changeToSlave(((DLegerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getId()); } else { startProcessorByHa(); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); @@ -1059,9 +1059,9 @@ public class BrokerController { } } - public void changeToSlave() { + public void changeToSlave(int brokerId) { //change the role - brokerConfig.setBrokerId(1); //TO DO check + brokerConfig.setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check messageStoreConfig.setBrokerRole(BrokerRole.SLAVE); //handle the scheduled service diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 9503fc8e34..3ba377bca6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -25,15 +25,18 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.dleger.DLegerCommitLog; public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private BrokerController brokerController; private DefaultMessageStore messageStore; + private DLegerCommitLog dLegerCommitLog; public DLegerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { this.brokerController = brokerController; this.messageStore = messageStore; + this.dLegerCommitLog = (DLegerCommitLog) messageStore.getCommitLog(); } @Override public void handle(long term, MemberState.Role role) { @@ -43,11 +46,11 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa switch (role) { case CANDIDATE: if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { - brokerController.changeToSlave(); + brokerController.changeToSlave(dLegerCommitLog.getId()); } break; case FOLLOWER: - brokerController.changeToSlave(); + brokerController.changeToSlave(dLegerCommitLog.getId()); break; case LEADER: while (messageStore.dispatchBehindBytes() != 0) { diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 7bef4f19a6..deb34f16c9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -58,6 +58,8 @@ public class DLegerCommitLog extends CommitLog { private final DLegerMmapFileStore dLegerFileStore; private final MmapFileList dLegerFileList; + private final int id; + private final MessageSerializer messageSerializer; @@ -71,6 +73,7 @@ public class DLegerCommitLog extends CommitLog { dLegerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); dLegerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); dLegerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); + id = Integer.valueOf(dLegerConfig.getSelfId().substring(1)) + 1; dLegerServer = new DLegerServer(dLegerConfig); dLegerFileStore = (DLegerMmapFileStore) dLegerServer.getdLegerStore(); DLegerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> { @@ -654,4 +657,8 @@ public class DLegerCommitLog extends CommitLog { public DLegerServer getdLegerServer() { return dLegerServer; } + + public int getId() { + return id; + } } From cbbd6ff0e62de1e213f2c8c3d386b1f51a890527 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 17:45:23 +0800 Subject: [PATCH 012/122] Polish role change logic --- .../rocketmq/broker/BrokerController.java | 13 ++- .../TransactionalMessageCheckService.java | 8 +- .../schedule/ScheduleMessageService.java | 85 ++++++++++++------- 3 files changed, 68 insertions(+), 38 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a852600665..a1471896ec 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -1060,6 +1060,8 @@ public class BrokerController { } public void changeToSlave(int brokerId) { + log.info("Begin to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId); + //change the role brokerConfig.setBrokerId(brokerId == 0 ? 1 : brokerId); //TO DO check messageStoreConfig.setBrokerRole(BrokerRole.SLAVE); @@ -1078,6 +1080,7 @@ public class BrokerController { } catch (Throwable ignored) { } + log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId); } @@ -1086,11 +1089,10 @@ public class BrokerController { if (role == BrokerRole.SLAVE) { return; } + log.info("Begin to change to master brokerName={}", brokerConfig.getBrokerName()); + //handle the slave synchronise handleSlaveSynchronize(role); - //change the role - brokerConfig.setBrokerId(0); //TO DO check - messageStoreConfig.setBrokerRole(role); //handle the scheduled service this.messageStore.handleScheduleMessageService(role); @@ -1098,11 +1100,16 @@ public class BrokerController { //handle the transactional service this.startProcessorByHa(); + //if the operations above are totally successful, we change to master + brokerConfig.setBrokerId(0); //TO DO check + messageStoreConfig.setBrokerRole(role); + try { this.registerBrokerAll(true, true, brokerConfig.isForceRegister()); } catch (Throwable ignored) { } + log.info("Finish to change to master brokerName={}", brokerConfig.getBrokerName()); } private void startProcessorByHa() { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index 5d515d6f11..bd98d0e78b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -39,7 +39,8 @@ public class TransactionalMessageCheckService extends ServiceThread { public void start() { if (started.compareAndSet(false, true)) { super.start(); - this.brokerController.getTransactionalMessageService().open(); + //no need to do this + //this.brokerController.getTransactionalMessageService().open(); } } @@ -47,8 +48,9 @@ public class TransactionalMessageCheckService extends ServiceThread { public void shutdown(boolean interrupt) { if (started.compareAndSet(true, false)) { super.shutdown(interrupt); - this.brokerController.getTransactionalMessageService().close(); - this.brokerController.getTransactionalMessageCheckListener().shutDown(); + //no need to do this + //this.brokerController.getTransactionalMessageService().close(); + //this.brokerController.getTransactionalMessageCheckListener().shutDown(); } } diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index e707463c79..50a48d4de9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -19,11 +19,11 @@ package org.apache.rocketmq.store.schedule; import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.Map.Entry; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.constant.LoggerName; @@ -38,6 +38,7 @@ import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; @@ -56,15 +57,15 @@ public class ScheduleMessageService extends ConfigManager { private final ConcurrentMap offsetTable = new ConcurrentHashMap(32); - - private final Timer timer = new Timer("ScheduleMessageTimerThread", true); - private final DefaultMessageStore defaultMessageStore; - + private final AtomicBoolean started = new AtomicBoolean(false); + private Timer timer; + private MessageStore writeMessageStore; private int maxDelayLevel; public ScheduleMessageService(final DefaultMessageStore defaultMessageStore) { this.defaultMessageStore = defaultMessageStore; + this.writeMessageStore = defaultMessageStore; } public static int queueId2DelayLevel(final int queueId) { @@ -75,10 +76,18 @@ public class ScheduleMessageService extends ConfigManager { return delayLevel - 1; } + /** + * @param writeMessageStore + * the writeMessageStore to set + */ + public void setWriteMessageStore(MessageStore writeMessageStore) { + this.writeMessageStore = writeMessageStore; + } + public void buildRunningStats(HashMap stats) { - Iterator> it = this.offsetTable.entrySet().iterator(); + Iterator> it = this.offsetTable.entrySet().iterator(); while (it.hasNext()) { - Entry next = it.next(); + Map.Entry next = it.next(); int queueId = delayLevel2QueueId(next.getKey()); long delayOffset = next.getValue(); long maxOffset = this.defaultMessageStore.getMaxOffsetInQueue(SCHEDULE_TOPIC, queueId); @@ -102,35 +111,45 @@ public class ScheduleMessageService extends ConfigManager { } public void start() { + if (started.compareAndSet(false, true)) { + this.timer = new Timer("ScheduleMessageTimerThread", true); + for (Map.Entry entry : this.delayLevelTable.entrySet()) { + Integer level = entry.getKey(); + Long timeDelay = entry.getValue(); + Long offset = this.offsetTable.get(level); + if (null == offset) { + offset = 0L; + } - for (Map.Entry entry : this.delayLevelTable.entrySet()) { - Integer level = entry.getKey(); - Long timeDelay = entry.getValue(); - Long offset = this.offsetTable.get(level); - if (null == offset) { - offset = 0L; - } - - if (timeDelay != null) { - this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME); - } - } - - this.timer.scheduleAtFixedRate(new TimerTask() { - - @Override - public void run() { - try { - ScheduleMessageService.this.persist(); - } catch (Throwable e) { - log.error("scheduleAtFixedRate flush exception", e); + if (timeDelay != null) { + this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME); } } - }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval()); + + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + if (started.get()) ScheduleMessageService.this.persist(); + } catch (Throwable e) { + log.error("scheduleAtFixedRate flush exception", e); + } + } + }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval()); + } } public void shutdown() { - this.timer.cancel(); + if (this.started.compareAndSet(true, false)) { + if (null != this.timer) + this.timer.cancel(); + } + + } + + public boolean isStarted() { + return started.get(); } public int getMaxDelayLevel() { @@ -214,7 +233,9 @@ public class ScheduleMessageService extends ConfigManager { @Override public void run() { try { - this.executeOnTimeup(); + if (isStarted()) { + this.executeOnTimeup(); + } } catch (Exception e) { // XXX: warn and notify me log.error("ScheduleMessageService, executeOnTimeup exception", e); @@ -285,7 +306,7 @@ public class ScheduleMessageService extends ConfigManager { try { MessageExtBrokerInner msgInner = this.messageTimeup(msgExt); PutMessageResult putMessageResult = - ScheduleMessageService.this.defaultMessageStore + ScheduleMessageService.this.writeMessageStore .putMessage(msgInner); if (putMessageResult != null From 41373641ff2b3d3b4794bfcd9409effbad3c1b53 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 13 Nov 2018 19:13:54 +0800 Subject: [PATCH 013/122] Remove non-existed brokerId of the same broker addr --- .../broker/dleger/DLegerRoleChangeHandler.java | 2 +- .../rocketmq/namesrv/routeinfo/RouteInfoManager.java | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 3ba377bca6..67498964ec 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -64,7 +64,7 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa } log.info("Finish handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); } catch (Throwable t) { - log.info("Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); + log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); } } } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index 00962ef246..ecd057a29a 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -128,6 +128,17 @@ public class RouteInfoManager { brokerData = new BrokerData(clusterName, brokerName, new HashMap()); this.brokerAddrTable.put(brokerName, brokerData); } + Map brokerAddrsMap = brokerData.getBrokerAddrs(); + //Switch slave to master: first remove <1, IP:PORT> in namesrv, then add <0, IP:PORT> + //The same IP:PORT must only have one record in brokerAddrTable + Iterator> it = brokerAddrsMap.entrySet().iterator(); + while (it.hasNext()) { + Entry item = it.next(); + if (null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey()) { + it.remove(); + } + } + String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr); registerFirst = registerFirst || (null == oldAddr); From 84bd6409b95f80b72f1a20875ad53b3ecb8e5bf0 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 14 Nov 2018 14:02:55 +0800 Subject: [PATCH 014/122] Reduce slave sync interval --- .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a1471896ec..52eb5003f3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -1049,7 +1049,7 @@ public class BrokerController { log.error("ScheduledTask SlaveSynchronize syncAll error.", e); } } - }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + }, 1000 * 3, 1000 * 3, TimeUnit.MILLISECONDS); } else { //handle the slave synchronise if (null != slaveSyncFuture) { From 5a19e6eee3d82723acaaf7fb9198274fb2cf2cf7 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 14 Nov 2018 15:54:08 +0800 Subject: [PATCH 015/122] Fix reput problem --- .../rocketmq/store/DefaultMessageStore.java | 8 +++----- .../store/dleger/DLegerCommitLog.java | 20 ++++++++++++------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index b3c1e7ba27..231bc01104 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -1817,12 +1817,10 @@ public class DefaultMessageStore implements MessageStore { this.reputFromOffset += size; } else { doNext = false; - if (DefaultMessageStore.this.brokerConfig.getBrokerId() == MixAll.MASTER_ID) { - log.error("[BUG]the master dispatch message to consume queue error, COMMITLOG OFFSET: {}", - this.reputFromOffset); + log.error("[BUG]dispatch message to consume queue error, COMMITLOG OFFSET: {}", + this.reputFromOffset); - this.reputFromOffset += result.getSize() - readSize; - } + this.reputFromOffset += result.getSize() - readSize; } } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index deb34f16c9..7a15c0a473 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -227,14 +227,20 @@ public class DLegerCommitLog extends CommitLog { final boolean readBody) { try { int bodyOffset = DLegerEntry.BODY_OFFSET; - byteBuffer.position(byteBuffer.position() + bodyOffset); - DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); - if (dispatchRequest.isSuccess()) { - dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); - } else if (dispatchRequest.getMsgSize() > 0) { - dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + int pos = byteBuffer.position(); + int magic = byteBuffer.getInt(); + if (magic == MmapFileList.BLANK_MAGIC_CODE) { + return new DispatchRequest(0, true); + } else { + byteBuffer.position(pos + bodyOffset); + DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); + if (dispatchRequest.isSuccess()) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } else if (dispatchRequest.getMsgSize() > 0) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } + return dispatchRequest; } - return dispatchRequest; } catch (Exception e) { } From 0c69acafa9d64c04b5c64c17148c2d604db5592b Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 14 Nov 2018 16:50:28 +0800 Subject: [PATCH 016/122] Get reput offset from logic queues --- .../apache/rocketmq/store/DefaultMessageStore.java | 14 ++++++++++++-- .../rocketmq/store/dleger/DLegerCommitLog.java | 9 +++++---- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 231bc01104..1c9af0493e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -227,10 +227,20 @@ public class DefaultMessageStore implements MessageStore { this.commitLog.start(); this.storeStatsService.start(); + //calculate the reput offset from the consume queue itself + long maxPhysicalPosInLogicQueue = commitLog.getMinOffset(); + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) { + maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset(); + } + } + } + if (this.getMessageStoreConfig().isDuplicationEnable()) { - this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset()); + this.reputMessageService.setReputFromOffset(Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset())); } else { - this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset()); + this.reputMessageService.setReputFromOffset(Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset())); } this.reputMessageService.start(); diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 7a15c0a473..c575aad90b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -248,16 +248,15 @@ public class DLegerCommitLog extends CommitLog { } public long getConfirmOffset() { - return this.confirmOffset; + return this.dLegerFileStore.getCommittedPos() == -1 ? getMaxOffset() + : this.dLegerFileStore.getCommittedPos(); } public void setConfirmOffset(long phyOffset) { - this.confirmOffset = phyOffset; + log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset); } - - private void notifyMessageArriving() { } @@ -438,6 +437,8 @@ public class DLegerCommitLog extends CommitLog { return dLegerFileList.getMinOffset(); } + + public SelectMappedBufferResult getMessage(final long offset, final int size) { int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, offset == 0); From 934267493e60aa348761c879516f734b5f20e91d Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 14 Nov 2018 17:28:29 +0800 Subject: [PATCH 017/122] Polish maxPhysicOffset for logic queue --- .../java/org/apache/rocketmq/store/ConsumeQueue.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index f5a1250d47..af8445275b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -106,7 +106,7 @@ public class ConsumeQueue { if (offset >= 0 && size > 0) { mappedFileOffset = i + CQ_STORE_UNIT_SIZE; - this.maxPhysicOffset = offset; + this.maxPhysicOffset = offset + size; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; } @@ -224,7 +224,7 @@ public class ConsumeQueue { int logicFileSize = this.mappedFileSize; - this.maxPhysicOffset = phyOffet - 1; + this.maxPhysicOffset = phyOffet; long maxExtAddr = 1; while (true) { MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); @@ -249,7 +249,7 @@ public class ConsumeQueue { mappedFile.setWrotePosition(pos); mappedFile.setCommittedPosition(pos); mappedFile.setFlushedPosition(pos); - this.maxPhysicOffset = offset; + this.maxPhysicOffset = offset + size; // This maybe not take effect, when not every consume queue has extend file. if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; @@ -267,7 +267,7 @@ public class ConsumeQueue { mappedFile.setWrotePosition(pos); mappedFile.setCommittedPosition(pos); mappedFile.setFlushedPosition(pos); - this.maxPhysicOffset = offset; + this.maxPhysicOffset = offset + size; if (isExtAddr(tagsCode)) { maxExtAddr = tagsCode; } @@ -420,7 +420,7 @@ public class ConsumeQueue { private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, final long cqOffset) { - if (offset <= this.maxPhysicOffset) { + if (offset + size <= this.maxPhysicOffset) { return true; } @@ -464,7 +464,7 @@ public class ConsumeQueue { ); } } - this.maxPhysicOffset = offset; + this.maxPhysicOffset = offset + size; return mappedFile.appendMessage(this.byteBufferIndex.array()); } return false; From ffc733a414d4ae10a1d4a25194d15ebc0a6ca3f3 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 19 Nov 2018 16:56:34 +0800 Subject: [PATCH 018/122] Polish log for reput offset --- .../java/org/apache/rocketmq/store/DefaultMessageStore.java | 3 +++ .../org/apache/rocketmq/store/dleger/DLegerCommitLog.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 1c9af0493e..ecf0cbd306 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -237,6 +237,9 @@ public class DefaultMessageStore implements MessageStore { } } + log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", + maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); + if (this.getMessageStoreConfig().isDuplicationEnable()) { this.reputMessageService.setReputFromOffset(Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset())); } else { diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index c575aad90b..3a5b78eb3f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -117,7 +117,11 @@ public class DLegerCommitLog extends CommitLog { } public long getMaxOffset() { - return this.dLegerFileList.getMaxWrotePosition(); + if (this.dLegerFileStore.getCommittedPos() != -1) { + return dLegerFileStore.getCommittedPos(); + } else { + return this.dLegerFileList.getMaxWrotePosition(); + } } public long remainHowManyDataToCommit() { From 866e3090393fe4a091364f4889af69ed0643b527 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 20 Nov 2018 15:06:46 +0800 Subject: [PATCH 019/122] Make role change handler asynchronous --- .../dleger/DLegerRoleChangeHandler.java | 74 +++++++++++++------ 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index 67498964ec..e1d8e7e968 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -16,9 +16,13 @@ */ package org.apache.rocketmq.broker.dleger; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.dleger.DLegerLeaderElector; +import org.apache.rocketmq.dleger.DLegerServer; import org.apache.rocketmq.dleger.MemberState; import org.apache.rocketmq.dleger.utils.UtilAll; import org.apache.rocketmq.logging.InternalLogger; @@ -30,41 +34,63 @@ import org.apache.rocketmq.store.dleger.DLegerCommitLog; public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_")); private BrokerController brokerController; private DefaultMessageStore messageStore; private DLegerCommitLog dLegerCommitLog; + private DLegerServer dLegerServer; public DLegerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { this.brokerController = brokerController; this.messageStore = messageStore; this.dLegerCommitLog = (DLegerCommitLog) messageStore.getCommitLog(); + this.dLegerServer = dLegerCommitLog.getdLegerServer(); } @Override public void handle(long term, MemberState.Role role) { - long start = System.currentTimeMillis(); - try { - log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); - switch (role) { - case CANDIDATE: - if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { - brokerController.changeToSlave(dLegerCommitLog.getId()); + Runnable runnable = new Runnable() { + @Override public void run() { + long start = System.currentTimeMillis(); + try { + log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); + switch (role) { + case CANDIDATE: + if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { + brokerController.changeToSlave(dLegerCommitLog.getId()); + } + break; + case FOLLOWER: + brokerController.changeToSlave(dLegerCommitLog.getId()); + break; + case LEADER: + while (dLegerServer.getMemberState().isLeader() + && (dLegerServer.getdLegerStore().getLegerEndIndex() != dLegerServer.getdLegerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { + UtilAll.sleep(100); + } + boolean succ = dLegerServer.getMemberState().isLeader() + && dLegerServer.getdLegerStore().getLegerEndIndex() == dLegerServer.getdLegerStore().getCommittedIndex() + && messageStore.dispatchBehindBytes() == 0; + if (succ) { + messageStore.recoverTopicQueueTable(); + brokerController.changeToMaster(BrokerRole.SYNC_MASTER); + } + log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); + break; + default: + break; } - break; - case FOLLOWER: - brokerController.changeToSlave(dLegerCommitLog.getId()); - break; - case LEADER: - while (messageStore.dispatchBehindBytes() != 0) { - Thread.sleep(100); - } - messageStore.recoverTopicQueueTable(); - brokerController.changeToMaster(BrokerRole.SYNC_MASTER); - break; - default: - break; + } catch (Throwable t) { + log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); + } } - log.info("Finish handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); - } catch (Throwable t) { - log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); - } + }; + executorService.submit(runnable); + } + + @Override public void startup() { + + } + + @Override public void shutdown() { + executorService.shutdown(); } } From b9ed0ab917ead249abfec9ea97332197754f7971 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 20 Nov 2018 15:47:29 +0800 Subject: [PATCH 020/122] Fix tests --- .../org/apache/rocketmq/store/DefaultMessageStore.java | 10 +++++++--- .../rocketmq/store/config/MessageStoreConfig.java | 2 +- .../org/apache/rocketmq/store/ConsumeQueueTest.java | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index ecf0cbd306..828088a817 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -239,12 +239,16 @@ public class DefaultMessageStore implements MessageStore { log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); - + long reputOffset; if (this.getMessageStoreConfig().isDuplicationEnable()) { - this.reputMessageService.setReputFromOffset(Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset())); + reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset()); } else { - this.reputMessageService.setReputFromOffset(Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset())); + reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset()); } + if (reputOffset < 0) { + reputOffset = 0; + } + this.reputMessageService.setReputFromOffset(reputOffset); this.reputMessageService.start(); if (!messageStoreConfig.isEnableDLegerCommitLog()) { diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 63c7c313fc..75d2c8eb3e 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -143,7 +143,7 @@ public class MessageStoreConfig { private int transientStorePoolSize = 5; private boolean fastFailIfNoBufferInStorePool = false; - private boolean enableDLegerCommitLog; + private boolean enableDLegerCommitLog = false; private String dLegerGroup; private String dLegerPeers; private String dLegerSelfId; diff --git a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java index 7e01b8513c..ede25ea56a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/ConsumeQueueTest.java @@ -30,9 +30,10 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.stats.BrokerStatsManager; -import static org.assertj.core.api.Assertions.assertThat; import org.junit.Test; +import static org.assertj.core.api.Assertions.assertThat; + public class ConsumeQueueTest { private static final String msg = "Once, there was a chance for me!"; From 7b429097e9784a39ade884f8bba4112fe798cf1b Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 20 Nov 2018 16:20:53 +0800 Subject: [PATCH 021/122] Review code and fix --- .../org/apache/rocketmq/broker/BrokerController.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 52eb5003f3..24453a59ab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -786,7 +786,7 @@ public class BrokerController { if (messageStoreConfig.isEnableDLegerCommitLog()) { changeToSlave(((DLegerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getId()); } else { - startProcessorByHa(); + startProcessorByHa(messageStoreConfig.getBrokerRole()); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); } @@ -1049,7 +1049,7 @@ public class BrokerController { log.error("ScheduledTask SlaveSynchronize syncAll error.", e); } } - }, 1000 * 3, 1000 * 3, TimeUnit.MILLISECONDS); + }, 1000 * 3, 1000 * 10, TimeUnit.MILLISECONDS); } else { //handle the slave synchronise if (null != slaveSyncFuture) { @@ -1098,7 +1098,7 @@ public class BrokerController { this.messageStore.handleScheduleMessageService(role); //handle the transactional service - this.startProcessorByHa(); + this.startProcessorByHa(BrokerRole.SYNC_MASTER); //if the operations above are totally successful, we change to master brokerConfig.setBrokerId(0); //TO DO check @@ -1112,8 +1112,8 @@ public class BrokerController { log.info("Finish to change to master brokerName={}", brokerConfig.getBrokerName()); } - private void startProcessorByHa() { - if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { + private void startProcessorByHa(BrokerRole role) { + if (BrokerRole.SLAVE != role) { if (this.transactionalMessageCheckService != null) { this.transactionalMessageCheckService.start(); } From f2bcae00f5a155774864fee83beb7997b8cab85b Mon Sep 17 00:00:00 2001 From: dongeforever Date: Fri, 23 Nov 2018 15:48:37 +0800 Subject: [PATCH 022/122] fix --- .../java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 3a5b78eb3f..2122211d53 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -275,6 +275,7 @@ public class DLegerCommitLog extends CommitLog { return beginTimeInDlegerLock; } + @Override public PutMessageResult putMessage(final MessageExtBrokerInner msg) { // Set the storage time msg.setStoreTimestamp(System.currentTimeMillis()); @@ -415,6 +416,7 @@ public class DLegerCommitLog extends CommitLog { } + @Override public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); } From 3e9799ba83af089457ce6303612c571832e1aa6a Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 27 Nov 2018 16:56:06 +0800 Subject: [PATCH 023/122] Use io.openmessaging instead --- .../dleger/DLegerRoleChangeHandler.java | 8 ++++---- store/pom.xml | 4 ++-- .../store/dleger/DLegerCommitLog.java | 20 +++++++++---------- .../DLegerSelectMappedBufferResult.java | 2 +- .../store/dleger/DLegerCommitlogTest.java | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java index e1d8e7e968..6c5fddcdfb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java @@ -21,10 +21,10 @@ import java.util.concurrent.Executors; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.dleger.DLegerLeaderElector; -import org.apache.rocketmq.dleger.DLegerServer; -import org.apache.rocketmq.dleger.MemberState; -import org.apache.rocketmq.dleger.utils.UtilAll; +import io.openmessaging.storage.dleger.DLegerLeaderElector; +import io.openmessaging.storage.dleger.DLegerServer; +import io.openmessaging.storage.dleger.MemberState; +import io.openmessaging.storage.dleger.utils.UtilAll; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; diff --git a/store/pom.xml b/store/pom.xml index 7de04fcae5..1c54d4dc79 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -29,8 +29,8 @@ - org.apache.rocketmq - rocketmq-dleger + io.openmessaging.storage + dleger 0.1-SNAPSHOT diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java index 2122211d53..9255ba1109 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java @@ -16,6 +16,16 @@ */ package org.apache.rocketmq.store.dleger; +import io.openmessaging.storage.dleger.DLegerConfig; +import io.openmessaging.storage.dleger.DLegerServer; +import io.openmessaging.storage.dleger.entry.DLegerEntry; +import io.openmessaging.storage.dleger.protocol.AppendEntryRequest; +import io.openmessaging.storage.dleger.protocol.AppendEntryResponse; +import io.openmessaging.storage.dleger.protocol.DLegerResponseCode; +import io.openmessaging.storage.dleger.store.file.DLegerMmapFileStore; +import io.openmessaging.storage.dleger.store.file.MmapFile; +import io.openmessaging.storage.dleger.store.file.MmapFileList; +import io.openmessaging.storage.dleger.store.file.SelectMmapBufferResult; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.concurrent.CompletableFuture; @@ -27,16 +37,6 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.sysflag.MessageSysFlag; -import org.apache.rocketmq.dleger.DLegerConfig; -import org.apache.rocketmq.dleger.DLegerServer; -import org.apache.rocketmq.dleger.entry.DLegerEntry; -import org.apache.rocketmq.dleger.protocol.AppendEntryRequest; -import org.apache.rocketmq.dleger.protocol.AppendEntryResponse; -import org.apache.rocketmq.dleger.protocol.DLegerResponseCode; -import org.apache.rocketmq.dleger.store.file.DLegerMmapFileStore; -import org.apache.rocketmq.dleger.store.file.MmapFile; -import org.apache.rocketmq.dleger.store.file.MmapFileList; -import org.apache.rocketmq.dleger.store.file.SelectMmapBufferResult; import org.apache.rocketmq.store.AppendMessageResult; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.CommitLog; diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java index 2d21af4a57..3f6b235ce8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java +++ b/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java @@ -17,7 +17,7 @@ package org.apache.rocketmq.store.dleger; -import org.apache.rocketmq.dleger.store.file.SelectMmapBufferResult; +import io.openmessaging.storage.dleger.store.file.SelectMmapBufferResult; import org.apache.rocketmq.store.SelectMappedBufferResult; public class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java index be606654d7..ed3397a4e3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java @@ -8,7 +8,7 @@ import java.util.UUID; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.dleger.DLegerServer; +import io.openmessaging.storage.dleger.DLegerServer; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; From d30d0e85caedf36528934936c2417226c1ca5305 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 3 Dec 2018 14:30:20 +0800 Subject: [PATCH 024/122] Polish DLedgerCommitlog, and rename dleger to dledger --- .../rocketmq/broker/BrokerController.java | 10 +- .../DLedgerRoleChangeHandler.java} | 18 +-- .../org/apache/rocketmq/store/CommitLog.java | 1 + .../rocketmq/store/DefaultMessageStore.java | 4 +- .../DLedgerCommitLog.java} | 109 +++++++----------- .../DLedgerSelectMappedBufferResult.java} | 6 +- .../DLedgerCommitlogTest.java} | 6 +- .../ProduceAndConsumeTest.java | 2 +- 8 files changed, 68 insertions(+), 88 deletions(-) rename broker/src/main/java/org/apache/rocketmq/broker/{dleger/DLegerRoleChangeHandler.java => dledger/DLedgerRoleChangeHandler.java} (88%) rename store/src/main/java/org/apache/rocketmq/store/{dleger/DLegerCommitLog.java => dledger/DLedgerCommitLog.java} (90%) rename store/src/main/java/org/apache/rocketmq/store/{dleger/DLegerSelectMappedBufferResult.java => dledger/DLedgerSelectMappedBufferResult.java} (85%) rename store/src/test/java/org/apache/rocketmq/store/{dleger/DLegerCommitlogTest.java => dledger/DLedgerCommitlogTest.java} (96%) rename test/src/test/java/org/apache/rocketmq/test/base/{dleger => dledger}/ProduceAndConsumeTest.java (99%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 24453a59ab..72cbbc1cc0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -39,7 +39,7 @@ import org.apache.rocketmq.broker.client.DefaultConsumerIdsChangeListener; import org.apache.rocketmq.broker.client.ProducerManager; import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager; -import org.apache.rocketmq.broker.dleger.DLegerRoleChangeHandler; +import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler; import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filtersrv.FilterServerManager; @@ -99,7 +99,7 @@ import org.apache.rocketmq.store.MessageArrivingListener; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.apache.rocketmq.store.dleger.DLegerCommitLog; +import org.apache.rocketmq.store.dledger.DLedgerCommitLog; import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -236,8 +236,8 @@ public class BrokerController { new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); if (messageStoreConfig.isEnableDLegerCommitLog()) { - DLegerRoleChangeHandler roleChangeHandler = new DLegerRoleChangeHandler(this, (DefaultMessageStore) messageStore); - ((DLegerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLegerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); + DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); + ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLegerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin @@ -784,7 +784,7 @@ public class BrokerController { } if (messageStoreConfig.isEnableDLegerCommitLog()) { - changeToSlave(((DLegerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getId()); + changeToSlave(((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getId()); } else { startProcessorByHa(messageStoreConfig.getBrokerRole()); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java similarity index 88% rename from broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java rename to broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index 6c5fddcdfb..3a2e19cb96 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dleger/DLegerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.broker.dleger; +package org.apache.rocketmq.broker.dledger; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -29,21 +29,21 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.BrokerRole; -import org.apache.rocketmq.store.dleger.DLegerCommitLog; +import org.apache.rocketmq.store.dledger.DLedgerCommitLog; -public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { +public class DLedgerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_")); private BrokerController brokerController; private DefaultMessageStore messageStore; - private DLegerCommitLog dLegerCommitLog; + private DLedgerCommitLog dLedgerCommitLog; private DLegerServer dLegerServer; - public DLegerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { + public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { this.brokerController = brokerController; this.messageStore = messageStore; - this.dLegerCommitLog = (DLegerCommitLog) messageStore.getCommitLog(); - this.dLegerServer = dLegerCommitLog.getdLegerServer(); + this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); + this.dLegerServer = dLedgerCommitLog.getdLegerServer(); } @Override public void handle(long term, MemberState.Role role) { @@ -55,11 +55,11 @@ public class DLegerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHa switch (role) { case CANDIDATE: if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { - brokerController.changeToSlave(dLegerCommitLog.getId()); + brokerController.changeToSlave(dLedgerCommitLog.getId()); } break; case FOLLOWER: - brokerController.changeToSlave(dLegerCommitLog.getId()); + brokerController.changeToSlave(dLedgerCommitLog.getId()); break; case LEADER: while (dLegerServer.getMemberState().isLeader() diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index d10e30ba7e..2528ef2d43 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -474,6 +474,7 @@ public class CommitLog { } // Commitlog case files are deleted else { + log.warn("The commitlog files are deleted, and delete the consume queue files"); this.mappedFileQueue.setFlushedWhere(0); this.mappedFileQueue.setCommittedWhere(0); this.defaultMessageStore.destroyLogics(); diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 828088a817..91d51530ba 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -52,7 +52,7 @@ import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.apache.rocketmq.store.dleger.DLegerCommitLog; +import org.apache.rocketmq.store.dledger.DLedgerCommitLog; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.index.IndexService; import org.apache.rocketmq.store.index.QueryOffsetResult; @@ -119,7 +119,7 @@ public class DefaultMessageStore implements MessageStore { this.brokerStatsManager = brokerStatsManager; this.allocateMappedFileService = new AllocateMappedFileService(this); if (messageStoreConfig.isEnableDLegerCommitLog()) { - this.commitLog = new DLegerCommitLog(this); + this.commitLog = new DLedgerCommitLog(this); } else { this.commitLog = new CommitLog(this); } diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java similarity index 90% rename from store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java rename to store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 9255ba1109..3dc443a976 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.store.dleger; +package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dleger.DLegerConfig; import io.openmessaging.storage.dleger.DLegerServer; @@ -34,7 +34,6 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.store.AppendMessageResult; @@ -52,7 +51,7 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService; /** * Store all metadata downtime for recovery, data protection reliability */ -public class DLegerCommitLog extends CommitLog { +public class DLedgerCommitLog extends CommitLog { private final DLegerServer dLegerServer; private final DLegerConfig dLegerConfig; private final DLegerMmapFileStore dLegerFileStore; @@ -66,7 +65,7 @@ public class DLegerCommitLog extends CommitLog { private volatile long beginTimeInDlegerLock = 0; - public DLegerCommitLog(final DefaultMessageStore defaultMessageStore) { + public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); dLegerConfig = new DLegerConfig(); dLegerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); @@ -87,6 +86,7 @@ public class DLegerCommitLog extends CommitLog { } + @Override public boolean load() { /*boolean result = this.mappedFileQueue.load(); log.info("load commit log " + (result ? "OK" : "Failed")); @@ -94,6 +94,7 @@ public class DLegerCommitLog extends CommitLog { return true; } + @Override public void start() { dLegerServer.startup(); /* this.flushCommitLogService.start(); @@ -103,6 +104,7 @@ public class DLegerCommitLog extends CommitLog { }*/ } + @Override public void shutdown() { /* if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { this.commitLogService.shutdown(); @@ -111,11 +113,13 @@ public class DLegerCommitLog extends CommitLog { dLegerServer.shutdown(); } + @Override public long flush() { dLegerFileStore.flush(); return dLegerFileList.getFlushedWhere(); } + @Override public long getMaxOffset() { if (this.dLegerFileStore.getCommittedPos() != -1) { return dLegerFileStore.getCommittedPos(); @@ -124,14 +128,17 @@ public class DLegerCommitLog extends CommitLog { } } + @Override public long remainHowManyDataToCommit() { return dLegerFileList.remainHowManyDataToCommit(); } + @Override public long remainHowManyDataToFlush() { return dLegerFileList.remainHowManyDataToFlush(); } + @Override public int deleteExpiredFile( final long expiredTime, final int deleteFilesInterval, @@ -142,7 +149,7 @@ public class DLegerCommitLog extends CommitLog { } - private static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { + static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { private SelectMmapBufferResult sbr; public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { @@ -182,11 +189,13 @@ public class DLegerCommitLog extends CommitLog { /** * Read CommitLog data, use data replication */ + @Override public SelectMappedBufferResult getData(final long offset) { return this.getData(offset, offset == 0); } + @Override public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { if (offset >= dLegerFileStore.getCommittedPos()) { return null; @@ -205,28 +214,27 @@ public class DLegerCommitLog extends CommitLog { /** * When the normal exit, data recovery, all memory data have been flush */ - public void recoverNormally() { - - } - public void recoverAbnormally() { + @Override + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { } + @Override + public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + + } + + @Override public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC) { return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true); } - private void doNothingForDeadCode(final Object obj) { - if (obj != null) { - log.debug(String.valueOf(obj.hashCode())); - } - } - /** * check the message and returns the message size * * @return 0 Come the end of the file // >0 Normal messages // -1 Message checksum failure */ + @Override public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) { try { @@ -251,22 +259,19 @@ public class DLegerCommitLog extends CommitLog { return new DispatchRequest(-1, false /* success */); } + @Override public long getConfirmOffset() { return this.dLegerFileStore.getCommittedPos() == -1 ? getMaxOffset() : this.dLegerFileStore.getCommittedPos(); } + @Override public void setConfirmOffset(long phyOffset) { log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset); } - - private void notifyMessageArriving() { - - } - + @Override public boolean resetOffset(long offset) { - //return this.mappedFileQueue.resetOffset(offset); return false; } @@ -349,7 +354,7 @@ public class DLegerCommitLog extends CommitLog { case MessageSysFlag.TRANSACTION_NOT_TYPE: case MessageSysFlag.TRANSACTION_COMMIT_TYPE: // The next update ConsumeQueue information - DLegerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); + DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); break; default: break; @@ -408,43 +413,20 @@ public class DLegerCommitLog extends CommitLog { return putMessageResult; } - public void handleDiskFlush(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) { - - } - - public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) { - - } - @Override public PutMessageResult putMessages(final MessageExtBatch messageExtBatch) { return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); } - /** - * According to receive certain message or offset storage time if an error occurs, it returns -1 - */ - public long pickupStoreTimestamp(final long offset, final int size) { - if (offset >= this.getMinOffset()) { - SelectMappedBufferResult result = this.getMessage(offset, size); - if (null != result) { - try { - return result.getByteBuffer().getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION); - } finally { - result.release(); - } - } - } - - return -1; - } + @Override public long getMinOffset() { return dLegerFileList.getMinOffset(); } + @Override public SelectMappedBufferResult getMessage(final long offset, final int size) { int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, offset == 0); @@ -455,42 +437,40 @@ public class DLegerCommitLog extends CommitLog { return null; } + @Override public long rollNextFile(final long offset) { int mappedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); return offset + mappedFileSize - offset % mappedFileSize; } + @Override public HashMap getTopicQueueTable() { return topicQueueTable; } + @Override public void setTopicQueueTable(HashMap topicQueueTable) { this.topicQueueTable = topicQueueTable; } + @Override public void destroy() { //TO DO } + @Override public boolean appendData(long startOffset, byte[] data) { //TO DO return false; } + @Override public boolean retryDeleteFirstFile(final long intervalForcibly) { - //TO DO return false; } - public void removeQueueFromTopicQueueTable(final String topic, final int queueId) { - String key = topic + "-" + queueId; - synchronized (this) { - this.topicQueueTable.remove(key); - } - - log.info("removeQueueFromTopicQueueTable OK Topic: {} QueueId: {}", topic, queueId); - } + @Override public void checkSelf() { dLegerFileList.checkSelf(); } @@ -560,10 +540,10 @@ public class DLegerCommitLog extends CommitLog { keyBuilder.append(msgInner.getQueueId()); String key = keyBuilder.toString(); - Long queueOffset = DLegerCommitLog.this.topicQueueTable.get(key); + Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key); if (null == queueOffset) { queueOffset = 0L; - DLegerCommitLog.this.topicQueueTable.put(key, queueOffset); + DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset); } // Transaction messages that require special handling @@ -603,7 +583,7 @@ public class DLegerCommitLog extends CommitLog { // Exceeds the maximum message if (msgLen > this.maxMessageSize) { - DLegerCommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength + DLedgerCommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + bodyLength + ", maxMessageSize: " + this.maxMessageSize); return new EncodeResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED, null, key); } @@ -612,7 +592,7 @@ public class DLegerCommitLog extends CommitLog { // 1 TOTALSIZE this.msgStoreItemMemory.putInt(msgLen); // 2 MAGICCODE - this.msgStoreItemMemory.putInt(DLegerCommitLog.MESSAGE_MAGIC_CODE); + this.msgStoreItemMemory.putInt(DLedgerCommitLog.MESSAGE_MAGIC_CODE); // 3 BODYCRC this.msgStoreItemMemory.putInt(msgInner.getBodyCRC()); // 4 QUEUEID @@ -642,18 +622,17 @@ public class DLegerCommitLog extends CommitLog { this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); // 15 BODY this.msgStoreItemMemory.putInt(bodyLength); - if (bodyLength > 0) + if (bodyLength > 0) { this.msgStoreItemMemory.put(msgInner.getBody()); + } // 16 TOPIC this.msgStoreItemMemory.put((byte) topicLength); this.msgStoreItemMemory.put(topicData); // 17 PROPERTIES this.msgStoreItemMemory.putShort((short) propertiesLength); - if (propertiesLength > 0) + if (propertiesLength > 0) { this.msgStoreItemMemory.put(propertiesData); - - final long beginTimeMills = DLegerCommitLog.this.defaultMessageStore.now(); - + } byte[] data = new byte[msgLen]; this.msgStoreItemMemory.clear(); this.msgStoreItemMemory.get(data); diff --git a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java similarity index 85% rename from store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java rename to store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java index 3f6b235ce8..7f5e0cd184 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dleger/DLegerSelectMappedBufferResult.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.rocketmq.store.dleger; +package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dleger.store.file.SelectMmapBufferResult; import org.apache.rocketmq.store.SelectMappedBufferResult; -public class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { +public class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult { private SelectMmapBufferResult sbr; - public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { + public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) { super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); this.sbr = sbr; } diff --git a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java similarity index 96% rename from store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java rename to store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index ed3397a4e3..36acc53435 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dleger/DLegerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -1,4 +1,4 @@ -package org.apache.rocketmq.store.dleger; +package org.apache.rocketmq.store.dledger; import java.io.File; import java.nio.ByteBuffer; @@ -22,7 +22,7 @@ import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.junit.Assert; import org.junit.Test; -public class DLegerCommitlogTest extends StoreTestBase { +public class DLedgerCommitlogTest extends StoreTestBase { private DefaultMessageStore createMessageStore(String base, String group, String selfId, String peers, String leaderId) throws Exception { baseDirs.add(base); @@ -43,7 +43,7 @@ public class DLegerCommitlogTest extends StoreTestBase { }, new BrokerConfig()); if (leaderId != null) { - DLegerServer dLegerServer = ((DLegerCommitLog) defaultMessageStore.getCommitLog()).getdLegerServer(); + DLegerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLegerServer(); dLegerServer.getdLegerConfig().setEnableLeaderElector(false); if (selfId.equals(leaderId)) { dLegerServer.getMemberState().changeToLeader(-1); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java b/test/src/test/java/org/apache/rocketmq/test/base/dledger/ProduceAndConsumeTest.java similarity index 99% rename from test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java rename to test/src/test/java/org/apache/rocketmq/test/base/dledger/ProduceAndConsumeTest.java index 30ddff4f24..813447f21f 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/dleger/ProduceAndConsumeTest.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/dledger/ProduceAndConsumeTest.java @@ -1,4 +1,4 @@ -package org.apache.rocketmq.test.base.dleger; +package org.apache.rocketmq.test.base.dledger; import java.util.UUID; import org.apache.rocketmq.broker.BrokerController; From f8634b4b6dc87fde5fd4e1ea80ad8b91638c9973 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 3 Dec 2018 16:25:42 +0800 Subject: [PATCH 025/122] Divide old commitlog and dledger commitlog --- .../rocketmq/broker/BrokerController.java | 2 +- .../dledger/DLedgerRoleChangeHandler.java | 2 +- .../org/apache/rocketmq/store/CommitLog.java | 2 +- .../store/dledger/DLedgerCommitLog.java | 265 +++++++++++------- .../store/dledger/DLedgerCommitlogTest.java | 2 +- 5 files changed, 160 insertions(+), 113 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 72cbbc1cc0..4b3c0a4683 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -237,7 +237,7 @@ public class BrokerController { this.brokerConfig); if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); - ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLegerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); + ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index 3a2e19cb96..f9391f32b8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -43,7 +43,7 @@ public class DLedgerRoleChangeHandler implements DLegerLeaderElector.RoleChangeH this.brokerController = brokerController; this.messageStore = messageStore; this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); - this.dLegerServer = dLedgerCommitLog.getdLegerServer(); + this.dLegerServer = dLedgerCommitLog.getdLedgerServer(); } @Override public void handle(long term, MemberState.Role role) { diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 2528ef2d43..8dec7408d1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -48,7 +48,7 @@ public class CommitLog { protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); // End of file empty MAGIC CODE cbd43194 protected final static int BLANK_MAGIC_CODE = -875286124; - private final MappedFileQueue mappedFileQueue; + protected final MappedFileQueue mappedFileQueue; protected final DefaultMessageStore defaultMessageStore; private final FlushCommitLogService flushCommitLogService; diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 3dc443a976..c6f3c7db85 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.CommitLog; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.MappedFile; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; @@ -52,90 +53,146 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService; * Store all metadata downtime for recovery, data protection reliability */ public class DLedgerCommitLog extends CommitLog { - private final DLegerServer dLegerServer; - private final DLegerConfig dLegerConfig; - private final DLegerMmapFileStore dLegerFileStore; - private final MmapFileList dLegerFileList; + private final DLegerServer dLedgerServer; + private final DLegerConfig dLedgerConfig; + private final DLegerMmapFileStore dLedgerFileStore; + private final MmapFileList dLedgerFileList; + //The id identifies the broker role, 0 means master, others means slave private final int id; - - private final MessageSerializer messageSerializer; + private volatile long beginTimeInDledgerLock = 0; - private volatile long beginTimeInDlegerLock = 0; + //This offset separate the old commitlog from dledger commitlog + private long dividedCommitlogOffset = -1; + + + //The old commitlog should be deleted before the dledger commitlog + private final boolean originalDledgerEnableForceClean; public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); - dLegerConfig = new DLegerConfig(); - dLegerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); - dLegerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); - dLegerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); - dLegerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); - id = Integer.valueOf(dLegerConfig.getSelfId().substring(1)) + 1; - dLegerServer = new DLegerServer(dLegerConfig); - dLegerFileStore = (DLegerMmapFileStore) dLegerServer.getdLegerStore(); + dLedgerConfig = new DLegerConfig(); + dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable()); + dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); + dLedgerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); + dLedgerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); + dLedgerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); + dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog()); + originalDledgerEnableForceClean = dLedgerConfig.isEnableDiskForceClean(); + id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1; + dLedgerServer = new DLegerServer(dLedgerConfig); + dLedgerFileStore = (DLegerMmapFileStore) dLedgerServer.getdLegerStore(); DLegerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> { assert bodyOffset == DLegerEntry.BODY_OFFSET; buffer.position(buffer.position() + bodyOffset + MessageDecoder.PHY_POS_POSITION); buffer.putLong(entry.getPos() + bodyOffset); }; - dLegerFileStore.addAppendHook(appendHook); - dLegerFileList = dLegerFileStore.getDataFileList(); + dLedgerFileStore.addAppendHook(appendHook); + dLedgerFileList = dLedgerFileStore.getDataFileList(); this.messageSerializer = new MessageSerializer(defaultMessageStore.getMessageStoreConfig().getMaxMessageSize()); } @Override public boolean load() { - /*boolean result = this.mappedFileQueue.load(); - log.info("load commit log " + (result ? "OK" : "Failed")); - return result;*/ + boolean result = super.load(); + if(!result) { + return false; + } + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + if (mappedFile == null) { + return true; + } + dLedgerConfig.setEnableDiskForceClean(false); + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + int currentPos = 0; + boolean needWriteMagicCode = true; + while (true) { + byteBuffer.position(currentPos); + // 1 TOTAL SIZE + int totalSize = byteBuffer.getInt(); + int magicCode = byteBuffer.getInt(); + if (magicCode == BLANK_MAGIC_CODE) { + needWriteMagicCode = false; + break; + } + if (magicCode != MESSAGE_MAGIC_CODE) { + log.info("Recover old commitlog found a illegal magic code={}", magicCode); + break; + } + currentPos = currentPos + totalSize; + } + log.info("Recover old commitlog needWriteMagicCode={} pos={} file={}", needWriteMagicCode, currentPos, mappedFile.getFileName()); + if (needWriteMagicCode) { + byteBuffer.position(currentPos); + byteBuffer.putInt(mappedFile.getFileSize() - currentPos); + byteBuffer.putInt(BLANK_MAGIC_CODE); + mappedFile.flush(0); + } + dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); + if (dLedgerFileList.getMappedFiles().isEmpty()) { + log.info("Recover to set the initial offset the dledger commitlog dividedCommitlogOffset={}", dividedCommitlogOffset); + dLedgerFileList.getLastMappedFile(dividedCommitlogOffset); + } return true; } @Override public void start() { - dLegerServer.startup(); - /* this.flushCommitLogService.start(); - - if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { - this.commitLogService.start(); - }*/ + dLedgerServer.startup(); } @Override public void shutdown() { - /* if (defaultMessageStore.getMessageStoreConfig().isTransientStorePoolEnable()) { - this.commitLogService.shutdown(); - } - this.flushCommitLogService.shutdown();*/ - dLegerServer.shutdown(); + dLedgerServer.shutdown(); } @Override public long flush() { - dLegerFileStore.flush(); - return dLegerFileList.getFlushedWhere(); + dLedgerFileStore.flush(); + return dLedgerFileList.getFlushedWhere(); } @Override public long getMaxOffset() { - if (this.dLegerFileStore.getCommittedPos() != -1) { - return dLegerFileStore.getCommittedPos(); + if (this.dLedgerFileStore.getCommittedPos() != -1) { + return dLedgerFileStore.getCommittedPos(); } else { - return this.dLegerFileList.getMaxWrotePosition(); + return this.dLedgerFileList.getMaxWrotePosition(); } } + @Override + public long getMinOffset() { + if (mappedFileQueue.getMappedFiles().isEmpty()) { + return dLedgerFileList.getMinOffset(); + } + return mappedFileQueue.getMinOffset(); + } + + @Override + public long getConfirmOffset() { + return this.dLedgerFileStore.getCommittedPos() == -1 ? getMaxOffset() + : this.dLedgerFileStore.getCommittedPos(); + } + + @Override + public void setConfirmOffset(long phyOffset) { + log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset); + } + + + @Override public long remainHowManyDataToCommit() { - return dLegerFileList.remainHowManyDataToCommit(); + return dLedgerFileList.remainHowManyDataToCommit(); } @Override public long remainHowManyDataToFlush() { - return dLegerFileList.remainHowManyDataToFlush(); + return dLedgerFileList.remainHowManyDataToFlush(); } @Override @@ -145,25 +202,15 @@ public class DLedgerCommitLog extends CommitLog { final long intervalForcibly, final boolean cleanImmediately ) { - return 0; + int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); + if (mappedFileQueue.getMappedFiles().isEmpty()) { + dLedgerConfig.setEnableDiskForceClean(originalDledgerEnableForceClean); + } + //return 1 to prevent too much log in defaultMessageStore + return count > 0 ? count : 1; } - static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { - - private SelectMmapBufferResult sbr; - public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { - super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); - this.sbr = sbr; - } - - public synchronized void release() { - super.release(); - sbr.release(); - } - - } - public SelectMappedBufferResult convertSbr(SelectMmapBufferResult sbr) { if (sbr == null) { return null; @@ -174,7 +221,7 @@ public class DLedgerCommitLog extends CommitLog { } public SelectMmapBufferResult truncate(SelectMmapBufferResult sbr) { - long committedPos = dLegerFileStore.getCommittedPos(); + long committedPos = dLedgerFileStore.getCommittedPos(); if (sbr == null || sbr.getStartOffset() == committedPos) { return null; } @@ -186,22 +233,25 @@ public class DLedgerCommitLog extends CommitLog { } } - /** - * Read CommitLog data, use data replication - */ @Override public SelectMappedBufferResult getData(final long offset) { + if (offset < dividedCommitlogOffset) { + return super.getData(offset); + } return this.getData(offset, offset == 0); } @Override public SelectMappedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { - if (offset >= dLegerFileStore.getCommittedPos()) { + if (offset < dividedCommitlogOffset) { + return super.getData(offset, returnFirstOnNotFound); + } + if (offset >= dLedgerFileStore.getCommittedPos()) { return null; } - int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); - MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound); + int mappedFileSize = this.dLedgerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); SelectMmapBufferResult sbr = mappedFile.selectMappedBuffer(pos); @@ -211,17 +261,18 @@ public class DLedgerCommitLog extends CommitLog { return null; } - /** - * When the normal exit, data recovery, all memory data have been flush - */ @Override public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { - + if (dLedgerFileList.getMappedFiles().isEmpty()) { + super.recoverAbnormally(maxPhyOffsetOfConsumeQueue); + } } @Override public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { - + if (dLedgerFileList.getMappedFiles().isEmpty()) { + super.recoverNormally(maxPhyOffsetOfConsumeQueue); + } } @Override @@ -229,11 +280,6 @@ public class DLedgerCommitLog extends CommitLog { return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true); } - /** - * check the message and returns the message size - * - * @return 0 Come the end of the file // >0 Normal messages // -1 Message checksum failure - */ @Override public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) { @@ -259,25 +305,15 @@ public class DLedgerCommitLog extends CommitLog { return new DispatchRequest(-1, false /* success */); } - @Override - public long getConfirmOffset() { - return this.dLegerFileStore.getCommittedPos() == -1 ? getMaxOffset() - : this.dLegerFileStore.getCommittedPos(); - } - - @Override - public void setConfirmOffset(long phyOffset) { - log.warn("Should not set confirm offset {} for dleger commitlog", phyOffset); - } - @Override public boolean resetOffset(long offset) { + //currently, it seems resetOffset has no use return false; } @Override public long getBeginTimeInLock() { - return beginTimeInDlegerLock; + return beginTimeInDledgerLock; } @Override @@ -325,7 +361,7 @@ public class DLedgerCommitLog extends CommitLog { putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long queueOffset = -1; try { - beginTimeInDlegerLock = this.defaultMessageStore.getSystemClock().now(); + beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); //TO DO use buffer encodeResult = this.messageSerializer.serialize(msg); queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); @@ -339,10 +375,10 @@ public class DLedgerCommitLog extends CommitLog { } } else { AppendEntryRequest request = new AppendEntryRequest(); - request.setGroup(dLegerConfig.getGroup()); - request.setRemoteId(dLegerServer.getMemberState().getSelfId()); + request.setGroup(dLedgerConfig.getGroup()); + request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); request.setBody(encodeResult.data); - dlegerFuture = dLegerServer.handleAppend(request); + dlegerFuture = dLedgerServer.handleAppend(request); if (dlegerFuture.isDone() && dlegerFuture.get().getCode() != DLegerResponseCode.SUCCESS.getCode()) { //TO DO make sure the local store is ok appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); @@ -361,12 +397,12 @@ public class DLedgerCommitLog extends CommitLog { } } } - eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDlegerLock; + eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; } catch (Exception e) { log.error("Put message error", e); appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); } finally { - beginTimeInDlegerLock = 0; + beginTimeInDledgerLock = 0; putMessageLock.unlock(); } @@ -419,17 +455,14 @@ public class DLedgerCommitLog extends CommitLog { } - @Override - public long getMinOffset() { - return dLegerFileList.getMinOffset(); - } - - @Override public SelectMappedBufferResult getMessage(final long offset, final int size) { - int mappedFileSize = this.dLegerServer.getdLegerConfig().getMappedFileSizeForEntryData(); - MmapFile mappedFile = this.dLegerFileList.findMappedFileByOffset(offset, offset == 0); + if (offset < dividedCommitlogOffset) { + return getMessage(offset, size); + } + int mappedFileSize = this.dLedgerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); return convertSbr(mappedFile.selectMappedBuffer(pos, size)); @@ -455,30 +488,25 @@ public class DLedgerCommitLog extends CommitLog { @Override public void destroy() { - //TO DO + super.destroy(); + dLedgerFileList.destroy(); } @Override public boolean appendData(long startOffset, byte[] data) { - //TO DO + //the old ha service will invoke method, here to prevent it return false; } - @Override - public boolean retryDeleteFirstFile(final long intervalForcibly) { - return false; - } - - @Override public void checkSelf() { - dLegerFileList.checkSelf(); + dLedgerFileList.checkSelf(); } @Override public long lockTimeMills() { long diff = 0; - long begin = this.beginTimeInDlegerLock; + long begin = this.beginTimeInDledgerLock; if (begin > 0) { diff = this.defaultMessageStore.now() - begin; } @@ -646,11 +674,30 @@ public class DLedgerCommitLog extends CommitLog { } - public DLegerServer getdLegerServer() { - return dLegerServer; + static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { + + private SelectMmapBufferResult sbr; + public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { + super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); + this.sbr = sbr; + } + + public synchronized void release() { + super.release(); + sbr.release(); + } + + } + + public DLegerServer getdLedgerServer() { + return dLedgerServer; } public int getId() { return id; } + + public long getDividedCommitlogOffset() { + return dividedCommitlogOffset; + } } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 36acc53435..7d5d07b314 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -43,7 +43,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { }, new BrokerConfig()); if (leaderId != null) { - DLegerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLegerServer(); + DLegerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); dLegerServer.getdLegerConfig().setEnableLeaderElector(false); if (selfId.equals(leaderId)) { dLegerServer.getMemberState().changeToLeader(-1); From 77b252bd7cd89c4c09c4f4149a9b3434a3b9a22f Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 3 Dec 2018 16:27:40 +0800 Subject: [PATCH 026/122] Polish checkstyle --- .../org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index c6f3c7db85..02d96a8684 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -98,7 +98,7 @@ public class DLedgerCommitLog extends CommitLog { @Override public boolean load() { boolean result = super.load(); - if(!result) { + if (!result) { return false; } MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); From b31c17f9e2ea62606b57655526c9a83dbdddc982 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 3 Dec 2018 17:13:02 +0800 Subject: [PATCH 027/122] Load and recover for dledger commitlog --- .../org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 02d96a8684..5945b851b9 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -263,6 +263,8 @@ public class DLedgerCommitLog extends CommitLog { @Override public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + dLedgerFileStore.load(); + dLedgerFileStore.recover(); if (dLedgerFileList.getMappedFiles().isEmpty()) { super.recoverAbnormally(maxPhyOffsetOfConsumeQueue); } @@ -270,6 +272,8 @@ public class DLedgerCommitLog extends CommitLog { @Override public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { + dLedgerFileStore.load(); + dLedgerFileStore.recover(); if (dLedgerFileList.getMappedFiles().isEmpty()) { super.recoverNormally(maxPhyOffsetOfConsumeQueue); } From 58496c2f3996a264bc223d617b98f2bd53f5c7bf Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 4 Dec 2018 13:57:55 +0800 Subject: [PATCH 028/122] Rename dleger to dledger --- .../dledger/DLedgerRoleChangeHandler.java | 16 ++--- store/pom.xml | 2 +- .../store/dledger/DLedgerCommitLog.java | 60 ++++++++++--------- .../DLedgerSelectMappedBufferResult.java | 36 ----------- .../store/dledger/DLedgerCommitlogTest.java | 6 +- 5 files changed, 43 insertions(+), 77 deletions(-) delete mode 100644 store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index f9391f32b8..b1f3bdcb22 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -21,24 +21,24 @@ import java.util.concurrent.Executors; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; -import io.openmessaging.storage.dleger.DLegerLeaderElector; -import io.openmessaging.storage.dleger.DLegerServer; -import io.openmessaging.storage.dleger.MemberState; -import io.openmessaging.storage.dleger.utils.UtilAll; +import io.openmessaging.storage.dledger.DLedgerLeaderElector; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.MemberState; +import io.openmessaging.storage.dledger.utils.UtilAll; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.dledger.DLedgerCommitLog; -public class DLedgerRoleChangeHandler implements DLegerLeaderElector.RoleChangeHandler { +public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLegerRoleChangeHandler_")); private BrokerController brokerController; private DefaultMessageStore messageStore; private DLedgerCommitLog dLedgerCommitLog; - private DLegerServer dLegerServer; + private DLedgerServer dLegerServer; public DLedgerRoleChangeHandler(BrokerController brokerController, DefaultMessageStore messageStore) { this.brokerController = brokerController; this.messageStore = messageStore; @@ -63,11 +63,11 @@ public class DLedgerRoleChangeHandler implements DLegerLeaderElector.RoleChangeH break; case LEADER: while (dLegerServer.getMemberState().isLeader() - && (dLegerServer.getdLegerStore().getLegerEndIndex() != dLegerServer.getdLegerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { + && (dLegerServer.getdLedgerStore().getLedgerEndIndex() != dLegerServer.getdLedgerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { UtilAll.sleep(100); } boolean succ = dLegerServer.getMemberState().isLeader() - && dLegerServer.getdLegerStore().getLegerEndIndex() == dLegerServer.getdLegerStore().getCommittedIndex() + && dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex() && messageStore.dispatchBehindBytes() == 0; if (succ) { messageStore.recoverTopicQueueTable(); diff --git a/store/pom.xml b/store/pom.xml index 1c54d4dc79..0d7681d34f 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -30,7 +30,7 @@ io.openmessaging.storage - dleger + dledger 0.1-SNAPSHOT diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 5945b851b9..028d537959 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.store.dledger; -import io.openmessaging.storage.dleger.DLegerConfig; -import io.openmessaging.storage.dleger.DLegerServer; -import io.openmessaging.storage.dleger.entry.DLegerEntry; -import io.openmessaging.storage.dleger.protocol.AppendEntryRequest; -import io.openmessaging.storage.dleger.protocol.AppendEntryResponse; -import io.openmessaging.storage.dleger.protocol.DLegerResponseCode; -import io.openmessaging.storage.dleger.store.file.DLegerMmapFileStore; -import io.openmessaging.storage.dleger.store.file.MmapFile; -import io.openmessaging.storage.dleger.store.file.MmapFileList; -import io.openmessaging.storage.dleger.store.file.SelectMmapBufferResult; +import io.openmessaging.storage.dledger.DLedgerConfig; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.entry.DLedgerEntry; +import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; +import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; +import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +import io.openmessaging.storage.dledger.store.file.MmapFile; +import io.openmessaging.storage.dledger.store.file.MmapFileList; +import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.concurrent.CompletableFuture; @@ -53,9 +53,9 @@ import org.apache.rocketmq.store.schedule.ScheduleMessageService; * Store all metadata downtime for recovery, data protection reliability */ public class DLedgerCommitLog extends CommitLog { - private final DLegerServer dLedgerServer; - private final DLegerConfig dLedgerConfig; - private final DLegerMmapFileStore dLedgerFileStore; + private final DLedgerServer dLedgerServer; + private final DLedgerConfig dLedgerConfig; + private final DLedgerMmapFileStore dLedgerFileStore; private final MmapFileList dLedgerFileList; //The id identifies the broker role, 0 means master, others means slave @@ -73,7 +73,7 @@ public class DLedgerCommitLog extends CommitLog { public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); - dLedgerConfig = new DLegerConfig(); + dLedgerConfig = new DLedgerConfig(); dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable()); dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); dLedgerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); @@ -82,10 +82,10 @@ public class DLedgerCommitLog extends CommitLog { dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog()); originalDledgerEnableForceClean = dLedgerConfig.isEnableDiskForceClean(); id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1; - dLedgerServer = new DLegerServer(dLedgerConfig); - dLedgerFileStore = (DLegerMmapFileStore) dLedgerServer.getdLegerStore(); - DLegerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> { - assert bodyOffset == DLegerEntry.BODY_OFFSET; + dLedgerServer = new DLedgerServer(dLedgerConfig); + dLedgerFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); + DLedgerMmapFileStore.AppendHook appendHook = (entry, buffer, bodyOffset) -> { + assert bodyOffset == DLedgerEntry.BODY_OFFSET; buffer.position(buffer.position() + bodyOffset + MessageDecoder.PHY_POS_POSITION); buffer.putLong(entry.getPos() + bodyOffset); }; @@ -215,7 +215,7 @@ public class DLedgerCommitLog extends CommitLog { if (sbr == null) { return null; } else { - return new DLegerSelectMappedBufferResult(sbr); + return new DLedgerSelectMappedBufferResult(sbr); } } @@ -250,7 +250,7 @@ public class DLedgerCommitLog extends CommitLog { if (offset >= dLedgerFileStore.getCommittedPos()) { return null; } - int mappedFileSize = this.dLedgerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, returnFirstOnNotFound); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); @@ -288,7 +288,7 @@ public class DLedgerCommitLog extends CommitLog { public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) { try { - int bodyOffset = DLegerEntry.BODY_OFFSET; + int bodyOffset = DLedgerEntry.BODY_OFFSET; int pos = byteBuffer.position(); int magic = byteBuffer.getInt(); if (magic == MmapFileList.BLANK_MAGIC_CODE) { @@ -383,7 +383,7 @@ public class DLedgerCommitLog extends CommitLog { request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); request.setBody(encodeResult.data); dlegerFuture = dLedgerServer.handleAppend(request); - if (dlegerFuture.isDone() && dlegerFuture.get().getCode() != DLegerResponseCode.SUCCESS.getCode()) { + if (dlegerFuture.isDone() && dlegerFuture.get().getCode() != DLedgerResponseCode.SUCCESS.getCode()) { //TO DO make sure the local store is ok appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); } else { @@ -417,10 +417,10 @@ public class DLedgerCommitLog extends CommitLog { if (dlegerFuture != null) { try { AppendEntryResponse appendEntryResponse = dlegerFuture.get(3, TimeUnit.SECONDS); - switch (DLegerResponseCode.valueOf(appendEntryResponse.getCode())) { + switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { case SUCCESS: putMessageStatus = PutMessageStatus.PUT_OK; - long wroteOffset = appendEntryResponse.getPos() + DLegerEntry.BODY_OFFSET; + long wroteOffset = appendEntryResponse.getPos() + DLedgerEntry.BODY_OFFSET; ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, 0); @@ -465,7 +465,7 @@ public class DLedgerCommitLog extends CommitLog { if (offset < dividedCommitlogOffset) { return getMessage(offset, size); } - int mappedFileSize = this.dLedgerServer.getdLegerConfig().getMappedFileSizeForEntryData(); + int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0); if (mappedFile != null) { int pos = (int) (offset % mappedFileSize); @@ -678,22 +678,24 @@ public class DLedgerCommitLog extends CommitLog { } - static class DLegerSelectMappedBufferResult extends SelectMappedBufferResult { + public static class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult { private SelectMmapBufferResult sbr; - public DLegerSelectMappedBufferResult(SelectMmapBufferResult sbr) { + public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) { super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); this.sbr = sbr; } public synchronized void release() { super.release(); - sbr.release(); + if (sbr != null) { + sbr.release(); + } } } - public DLegerServer getdLedgerServer() { + public DLedgerServer getdLedgerServer() { return dLedgerServer; } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java deleted file mode 100644 index 7f5e0cd184..0000000000 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerSelectMappedBufferResult.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.store.dledger; - -import io.openmessaging.storage.dleger.store.file.SelectMmapBufferResult; -import org.apache.rocketmq.store.SelectMappedBufferResult; - -public class DLedgerSelectMappedBufferResult extends SelectMappedBufferResult { - - private SelectMmapBufferResult sbr; - public DLedgerSelectMappedBufferResult(SelectMmapBufferResult sbr) { - super(sbr.getStartOffset(), sbr.getByteBuffer(), sbr.getSize(), null); - this.sbr = sbr; - } - - @Override - public synchronized void release() { - super.release(); - sbr.release(); - } -} diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 7d5d07b314..57742856d3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -8,7 +8,7 @@ import java.util.UUID; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; -import io.openmessaging.storage.dleger.DLegerServer; +import io.openmessaging.storage.dledger.DLedgerServer; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; @@ -43,8 +43,8 @@ public class DLedgerCommitlogTest extends StoreTestBase { }, new BrokerConfig()); if (leaderId != null) { - DLegerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); - dLegerServer.getdLegerConfig().setEnableLeaderElector(false); + DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); + dLegerServer.getdLedgerConfig().setEnableLeaderElector(false); if (selfId.equals(leaderId)) { dLegerServer.getMemberState().changeToLeader(-1); } else { From 8f788c3d09d8ae279959be0a56df7f4854b4bd87 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 4 Dec 2018 16:02:52 +0800 Subject: [PATCH 029/122] Add reputoffset test for dledger commitlog test --- .../store/dledger/DLedgerCommitLog.java | 120 ++++++++---------- .../store/dledger/DLedgerCommitlogTest.java | 62 ++++++++- 2 files changed, 111 insertions(+), 71 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 028d537959..e923f879d5 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store.dledger; +import io.openmessaging.storage.dledger.AppendFuture; import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import io.openmessaging.storage.dledger.entry.DLedgerEntry; @@ -28,7 +29,6 @@ import io.openmessaging.storage.dledger.store.file.MmapFileList; import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; import java.nio.ByteBuffer; import java.util.HashMap; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageAccessor; @@ -357,54 +357,49 @@ public class DLedgerCommitLog extends CommitLog { } // Back to Results - AppendMessageResult appendResult = null; - PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; - CompletableFuture dlegerFuture = null; - EncodeResult encodeResult = null; - long eclipseTimeInLock = 0L; + PutMessageStatus putMessageStatus = null; + AppendMessageResult appendResult; + AppendFuture dledgerFuture; + EncodeResult encodeResult; + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config - long queueOffset = -1; + long eclipseTimeInLock; + long queueOffset; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - //TO DO use buffer encodeResult = this.messageSerializer.serialize(msg); queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); if (encodeResult.status != AppendMessageStatus.PUT_OK) { - appendResult = new AppendMessageResult(encodeResult.status); - switch (encodeResult.status) { - case PROPERTIES_SIZE_EXCEEDED: - case MESSAGE_SIZE_EXCEEDED: - putMessageStatus = PutMessageStatus.MESSAGE_ILLEGAL; - break; - } - } else { - AppendEntryRequest request = new AppendEntryRequest(); - request.setGroup(dLedgerConfig.getGroup()); - request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); - request.setBody(encodeResult.data); - dlegerFuture = dLedgerServer.handleAppend(request); - if (dlegerFuture.isDone() && dlegerFuture.get().getCode() != DLedgerResponseCode.SUCCESS.getCode()) { - //TO DO make sure the local store is ok - appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); - } else { - switch (tranType) { - case MessageSysFlag.TRANSACTION_PREPARED_TYPE: - case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: - break; - case MessageSysFlag.TRANSACTION_NOT_TYPE: - case MessageSysFlag.TRANSACTION_COMMIT_TYPE: - // The next update ConsumeQueue information - DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); - break; - default: - break; - } - } + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); } + AppendEntryRequest request = new AppendEntryRequest(); + request.setGroup(dLedgerConfig.getGroup()); + request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); + request.setBody(encodeResult.data); + dledgerFuture = (AppendFuture) dLedgerServer.handleAppend(request); + if (dledgerFuture.getPos() == -1) { + return new PutMessageResult(PutMessageStatus.OS_PAGECACHE_BUSY, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)); + } + long wroteOffset = dledgerFuture.getPos() + DLedgerEntry.BODY_OFFSET; + ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, eclipseTimeInLock); + switch (tranType) { + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: + break; + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: + // The next update ConsumeQueue information + DLedgerCommitLog.this.topicQueueTable.put(encodeResult.queueOffsetKey, queueOffset + 1); + break; + default: + break; + } } catch (Exception e) { log.error("Put message error", e); - appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR)); } finally { beginTimeInDledgerLock = 0; putMessageLock.unlock(); @@ -414,34 +409,27 @@ public class DLedgerCommitLog extends CommitLog { log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, appendResult); } - if (dlegerFuture != null) { - try { - AppendEntryResponse appendEntryResponse = dlegerFuture.get(3, TimeUnit.SECONDS); - switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { - case SUCCESS: - putMessageStatus = PutMessageStatus.PUT_OK; - long wroteOffset = appendEntryResponse.getPos() + DLedgerEntry.BODY_OFFSET; - ByteBuffer buffer = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); - String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); - appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.data.length, msgId, System.currentTimeMillis(), queueOffset, 0); - break; - case INCONSISTENT_LEADER: - case NOT_LEADER: - case LEADER_NOT_READY: - case DISK_FULL: - putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; - break; - case WAIT_QUORUM_ACK_TIMEOUT: - putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; - break; - case LEADER_PENDING_FULL: - putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; - break; - } - } catch (Exception ignored) { - putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; - appendResult = new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + try { + AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS); + switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { + case SUCCESS: + putMessageStatus = PutMessageStatus.PUT_OK; + break; + case INCONSISTENT_LEADER: + case NOT_LEADER: + case LEADER_NOT_READY: + case DISK_FULL: + putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; + break; + case WAIT_QUORUM_ACK_TIMEOUT: + putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; + break; + case LEADER_PENDING_FULL: + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; + break; } + } catch (Throwable ignored) { + putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; } PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 57742856d3..57f04ea6a3 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -12,19 +12,21 @@ import io.openmessaging.storage.dledger.DLedgerServer; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; +import org.apache.rocketmq.store.MappedFile; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.StoreTestBase; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.junit.Assert; import org.junit.Test; public class DLedgerCommitlogTest extends StoreTestBase { - private DefaultMessageStore createMessageStore(String base, String group, String selfId, String peers, String leaderId) throws Exception { + private DefaultMessageStore createMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort) throws Exception { baseDirs.add(base); MessageStoreConfig storeConfig = new MessageStoreConfig(); storeConfig.setMapedFileSizeCommitLog(1024 * 100); @@ -39,7 +41,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { storeConfig.setdLegerGroup(group); storeConfig.setdLegerPeers(peers); storeConfig.setdLegerSelfId(selfId); - DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLegerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { + DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLedgerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { }, new BrokerConfig()); if (leaderId != null) { @@ -52,17 +54,67 @@ public class DLedgerCommitlogTest extends StoreTestBase { } } + if (createAbort) { + String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir()); + File file = new File(fileName); + MappedFile.ensureDirOK(file.getParent()); + file.createNewFile(); + } defaultMessageStore.load(); defaultMessageStore.start(); return defaultMessageStore; } + + + @Test + public void testReputOffset() throws Exception { + + String base = createBaseDir(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + + { + DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); + Thread.sleep(1000); + String topic = UUID.randomUUID().toString(); + for (int i = 0; i < 1000; i++) { + MessageExtBrokerInner msgInner = buildMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(0); + PutMessageResult putMessageResult = messageStore.putMessage(msgInner); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + Thread.sleep(100); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + messageStore.shutdown(); + } + + { + //normal recover + DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + messageStore.shutdown(); + } + { + //normal recover + DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, true); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + messageStore.shutdown(); + } + } + + + @Test public void testPutAndGetMessage() throws Exception { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null); + DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); @@ -102,7 +154,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { public void testCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), group,"n0", peers, "n0"); + DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), group,"n0", peers, "n0", false); String topic = UUID.randomUUID().toString(); MessageExtBrokerInner msgInner = buildMessage(); @@ -117,7 +169,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createMessageStore(createBaseDir(), group,"n1", peers, "n0"); + DefaultMessageStore followerStore = createMessageStore(createBaseDir(), group,"n1", peers, "n0", false); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); From ed31e74f6846f437415b0bb7f66182335203f06a Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 4 Dec 2018 17:58:40 +0800 Subject: [PATCH 030/122] Add tests for mixed commitlog test --- .../org/apache/rocketmq/store/CommitLog.java | 2 +- .../store/dledger/DLedgerCommitLog.java | 122 ++++++++++-------- .../apache/rocketmq/store/StoreTestBase.java | 8 ++ .../store/dledger/DLedgerCommitlogTest.java | 76 ++--------- .../store/dledger/MessageStoreTestBase.java | 92 +++++++++++++ .../store/dledger/MixCommitlogTest.java | 52 ++++++++ 6 files changed, 231 insertions(+), 121 deletions(-) create mode 100644 store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java create mode 100644 store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 8dec7408d1..64437565e2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -456,7 +456,7 @@ public class CommitLog { } } } else { - log.info("recover physics file end, " + mappedFile.getFileName()); + log.info("recover physics file end, " + mappedFile.getFileName() + " pos=" + byteBuffer.position()); break; } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index e923f879d5..655c4de4c7 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -71,6 +71,9 @@ public class DLedgerCommitLog extends CommitLog { //The old commitlog should be deleted before the dledger commitlog private final boolean originalDledgerEnableForceClean; + + private boolean isInrecoveringOldCommitlog = false; + public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { super(defaultMessageStore); dLedgerConfig = new DLedgerConfig(); @@ -101,41 +104,7 @@ public class DLedgerCommitLog extends CommitLog { if (!result) { return false; } - MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); - if (mappedFile == null) { - return true; - } - dLedgerConfig.setEnableDiskForceClean(false); - ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); - int currentPos = 0; - boolean needWriteMagicCode = true; - while (true) { - byteBuffer.position(currentPos); - // 1 TOTAL SIZE - int totalSize = byteBuffer.getInt(); - int magicCode = byteBuffer.getInt(); - if (magicCode == BLANK_MAGIC_CODE) { - needWriteMagicCode = false; - break; - } - if (magicCode != MESSAGE_MAGIC_CODE) { - log.info("Recover old commitlog found a illegal magic code={}", magicCode); - break; - } - currentPos = currentPos + totalSize; - } - log.info("Recover old commitlog needWriteMagicCode={} pos={} file={}", needWriteMagicCode, currentPos, mappedFile.getFileName()); - if (needWriteMagicCode) { - byteBuffer.position(currentPos); - byteBuffer.putInt(mappedFile.getFileSize() - currentPos); - byteBuffer.putInt(BLANK_MAGIC_CODE); - mappedFile.flush(0); - } - dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); - if (dLedgerFileList.getMappedFiles().isEmpty()) { - log.info("Recover to set the initial offset the dledger commitlog dividedCommitlogOffset={}", dividedCommitlogOffset); - dLedgerFileList.getLastMappedFile(dividedCommitlogOffset); - } + return true; } @@ -261,22 +230,59 @@ public class DLedgerCommitLog extends CommitLog { return null; } - @Override - public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + private void recover(long maxPhyOffsetOfConsumeQueue, boolean lastOk) { dLedgerFileStore.load(); - dLedgerFileStore.recover(); - if (dLedgerFileList.getMappedFiles().isEmpty()) { + if (dLedgerFileList.getMappedFiles().size() > 0) { + dLedgerFileStore.recover(); + return; + } + //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog + isInrecoveringOldCommitlog = true; + if (lastOk) { + super.recoverNormally(maxPhyOffsetOfConsumeQueue); + } else { super.recoverAbnormally(maxPhyOffsetOfConsumeQueue); } + isInrecoveringOldCommitlog = false; + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + if (mappedFile == null) { + return; + } + dLedgerConfig.setEnableDiskForceClean(false); + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + byteBuffer.position(mappedFile.getWrotePosition()); + boolean needWriteMagicCode = true; + // 1 TOTAL SIZE + byteBuffer.getInt(); //size + int magicCode = byteBuffer.getInt(); + if (magicCode == CommitLog.BLANK_MAGIC_CODE) { + needWriteMagicCode = false; + } else { + log.info("Recover old commitlog found a illegal magic code={}", magicCode); + } + dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); + log.info("Recover old commitlog needWriteMagicCode={} pos={} file={} dividedCommitlogOffset={}", needWriteMagicCode, mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(), mappedFile.getFileName(), dividedCommitlogOffset); + if (needWriteMagicCode) { + byteBuffer.position(mappedFile.getWrotePosition()); + byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition()); + byteBuffer.putInt(BLANK_MAGIC_CODE); + mappedFile.flush(0); + mappedFile.setWrotePosition(mappedFile.getFileSize()); + mappedFile.setCommittedPosition(mappedFile.getFileSize()); + mappedFile.setFlushedPosition(mappedFile.getFileSize()); + } + dLedgerFileList.getLastMappedFile(dividedCommitlogOffset); + log.info("Will set the initial commitlog offset={} for dledger", dividedCommitlogOffset); } @Override public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { - dLedgerFileStore.load(); - dLedgerFileStore.recover(); - if (dLedgerFileList.getMappedFiles().isEmpty()) { - super.recoverNormally(maxPhyOffsetOfConsumeQueue); - } + recover(maxPhyOffsetOfConsumeQueue, true); + } + + @Override + public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { + recover(maxPhyOffsetOfConsumeQueue, false); } @Override @@ -287,23 +293,31 @@ public class DLedgerCommitLog extends CommitLog { @Override public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC, final boolean readBody) { + if (isInrecoveringOldCommitlog) { + return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); + } try { int bodyOffset = DLedgerEntry.BODY_OFFSET; int pos = byteBuffer.position(); int magic = byteBuffer.getInt(); + //In dledger, this field is size, it must be gt 0, so it could prevent collision + int magicOld = byteBuffer.getInt(); + if (magicOld == CommitLog.BLANK_MAGIC_CODE || magicOld == CommitLog.MESSAGE_MAGIC_CODE) { + byteBuffer.position(pos); + return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); + } if (magic == MmapFileList.BLANK_MAGIC_CODE) { return new DispatchRequest(0, true); - } else { - byteBuffer.position(pos + bodyOffset); - DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); - if (dispatchRequest.isSuccess()) { - dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); - } else if (dispatchRequest.getMsgSize() > 0) { - dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); - } - return dispatchRequest; } - } catch (Exception e) { + byteBuffer.position(pos + bodyOffset); + DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody); + if (dispatchRequest.isSuccess()) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } else if (dispatchRequest.getMsgSize() > 0) { + dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset); + } + return dispatchRequest; + } catch (Throwable ignored) { } return new DispatchRequest(-1, false /* success */); diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java index 9e09d8132d..aa2919d821 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -9,6 +9,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.junit.After; public class StoreTestBase { @@ -51,6 +52,13 @@ public class StoreTestBase { return baseDir; } + public static boolean makeSureFileExists(String fileName) throws Exception { + File file = new File(fileName); + MappedFile.ensureDirOK(file.getParent()); + return file.createNewFile(); + } + + public static void deleteFile(String fileName) { deleteFile(new File(fileName)); } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 57f04ea6a3..28a1d8ea10 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -1,91 +1,34 @@ package org.apache.rocketmq.store.dledger; -import java.io.File; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; -import io.openmessaging.storage.dledger.DLedgerServer; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; -import org.apache.rocketmq.store.MappedFile; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; -import org.apache.rocketmq.store.StoreTestBase; -import org.apache.rocketmq.store.config.FlushDiskType; -import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.apache.rocketmq.store.config.StorePathConfigHelper; -import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.junit.Assert; import org.junit.Test; -public class DLedgerCommitlogTest extends StoreTestBase { - - private DefaultMessageStore createMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort) throws Exception { - baseDirs.add(base); - MessageStoreConfig storeConfig = new MessageStoreConfig(); - storeConfig.setMapedFileSizeCommitLog(1024 * 100); - storeConfig.setMapedFileSizeConsumeQueue(1024); - storeConfig.setMaxHashSlotNum(100); - storeConfig.setMaxIndexNum(100 * 10); - storeConfig.setStorePathRootDir(base); - storeConfig.setStorePathCommitLog(base + File.separator + "commitlog"); - storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH); - - storeConfig.setEnableDLegerCommitLog(true); - storeConfig.setdLegerGroup(group); - storeConfig.setdLegerPeers(peers); - storeConfig.setdLegerSelfId(selfId); - DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLedgerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { - - }, new BrokerConfig()); - if (leaderId != null) { - DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); - dLegerServer.getdLedgerConfig().setEnableLeaderElector(false); - if (selfId.equals(leaderId)) { - dLegerServer.getMemberState().changeToLeader(-1); - } else { - dLegerServer.getMemberState().changeToFollower(-1, leaderId); - } - - } - if (createAbort) { - String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir()); - File file = new File(fileName); - MappedFile.ensureDirOK(file.getParent()); - file.createNewFile(); - } - defaultMessageStore.load(); - defaultMessageStore.start(); - return defaultMessageStore; - } - +public class DLedgerCommitlogTest extends MessageStoreTestBase { @Test public void testReputOffset() throws Exception { - String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); { - DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); - for (int i = 0; i < 1000; i++) { - MessageExtBrokerInner msgInner = buildMessage(); - msgInner.setTopic(topic); - msgInner.setQueueId(0); - PutMessageResult putMessageResult = messageStore.putMessage(msgInner); - Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); - Assert.assertEquals(i, putMessageResult.getAppendMessageResult().getLogicsOffset()); - } + doPutMessages(messageStore, topic, 0, 1000, 0); Thread.sleep(100); Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); @@ -95,13 +38,14 @@ public class DLedgerCommitlogTest extends StoreTestBase { { //normal recover - DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); messageStore.shutdown(); } + { - //normal recover - DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, true); + //abnormal recover + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); messageStore.shutdown(); } @@ -114,7 +58,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore messageStore = createMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); @@ -154,7 +98,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { public void testCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore leaderStore = createMessageStore(createBaseDir(), group,"n0", peers, "n0", false); + DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false); String topic = UUID.randomUUID().toString(); MessageExtBrokerInner msgInner = buildMessage(); @@ -169,7 +113,7 @@ public class DLedgerCommitlogTest extends StoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createMessageStore(createBaseDir(), group,"n1", peers, "n0", false); + DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java new file mode 100644 index 0000000000..d671287d89 --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -0,0 +1,92 @@ +package org.apache.rocketmq.store.dledger; + +import io.openmessaging.storage.dledger.DLedgerServer; +import java.io.File; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.StoreTestBase; +import org.apache.rocketmq.store.config.FlushDiskType; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.stats.BrokerStatsManager; +import org.junit.Assert; + +public class MessageStoreTestBase extends StoreTestBase { + + protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort) throws Exception { + baseDirs.add(base); + MessageStoreConfig storeConfig = new MessageStoreConfig(); + storeConfig.setMapedFileSizeCommitLog(1024 * 100); + storeConfig.setMapedFileSizeConsumeQueue(1024); + storeConfig.setMaxHashSlotNum(100); + storeConfig.setMaxIndexNum(100 * 10); + storeConfig.setStorePathRootDir(base); + storeConfig.setStorePathCommitLog(base + File.separator + "commitlog"); + storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH); + + storeConfig.setEnableDLegerCommitLog(true); + storeConfig.setdLegerGroup(group); + storeConfig.setdLegerPeers(peers); + storeConfig.setdLegerSelfId(selfId); + DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLedgerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { + + }, new BrokerConfig()); + if (leaderId != null) { + DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); + dLegerServer.getdLedgerConfig().setEnableLeaderElector(false); + if (selfId.equals(leaderId)) { + dLegerServer.getMemberState().changeToLeader(-1); + } else { + dLegerServer.getMemberState().changeToFollower(-1, leaderId); + } + + } + if (createAbort) { + String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir()); + makeSureFileExists(fileName); + } + Assert.assertTrue(defaultMessageStore.load()); + defaultMessageStore.start(); + return defaultMessageStore; + } + + + protected DefaultMessageStore createMessageStore(String base, boolean createAbort) throws Exception { + baseDirs.add(base); + MessageStoreConfig storeConfig = new MessageStoreConfig(); + storeConfig.setMapedFileSizeCommitLog(1024 * 100); + storeConfig.setMapedFileSizeConsumeQueue(1024); + storeConfig.setMaxHashSlotNum(100); + storeConfig.setMaxIndexNum(100 * 10); + storeConfig.setStorePathRootDir(base); + storeConfig.setStorePathCommitLog(base + File.separator + "commitlog"); + storeConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH); + DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("CommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { + + }, new BrokerConfig()); + + if (createAbort) { + String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir()); + makeSureFileExists(fileName); + } + Assert.assertTrue(defaultMessageStore.load()); + defaultMessageStore.start(); + return defaultMessageStore; + } + + protected void doPutMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) { + for (int i = 0; i < num; i++) { + MessageExtBrokerInner msgInner = buildMessage(); + msgInner.setTopic(topic); + msgInner.setQueueId(queueId); + PutMessageResult putMessageResult = messageStore.putMessage(msgInner); + Assert.assertEquals(PutMessageStatus.PUT_OK, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(beginLogicsOffset + i, putMessageResult.getAppendMessageResult().getLogicsOffset()); + } + } + +} diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java new file mode 100644 index 0000000000..cba9fe9e0f --- /dev/null +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -0,0 +1,52 @@ +package org.apache.rocketmq.store.dledger; + +import java.util.UUID; +import org.apache.rocketmq.store.DefaultMessageStore; +import org.junit.Assert; +import org.junit.Test; + +public class MixCommitlogTest extends DLedgerCommitlogTest { + + + @Test + public void testPutAndGet() throws Exception { + String base = createBaseDir(); + String topic = UUID.randomUUID().toString(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + + long dividedOffset; + { + DefaultMessageStore originalStore = createMessageStore(base, false); + doPutMessages(originalStore, topic, 0, 1000, 0); + Thread.sleep(500); + Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, originalStore.dispatchBehindBytes()); + dividedOffset = originalStore.getCommitLog().getMaxOffset(); + dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + originalStore.shutdown(); + } + { + DefaultMessageStore recoverOriginalStore = createMessageStore(base, true); + Thread.sleep(500); + Assert.assertEquals(0, recoverOriginalStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, recoverOriginalStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, recoverOriginalStore.dispatchBehindBytes()); + recoverOriginalStore.shutdown(); + } + { + DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); + Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); + Thread.sleep(2000); + doPutMessages(dledgerStore, topic, 0, 1000, 1000); + Thread.sleep(500); + Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + dledgerStore.shutdown(); + } + + } +} From 927385be33bbe2b5eb404364d1bdfca4ed646fb5 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 4 Dec 2018 19:05:19 +0800 Subject: [PATCH 031/122] Enhance tests for mixed commitlog --- .../store/dledger/DLedgerCommitLog.java | 9 +++++++-- .../store/dledger/DLedgerCommitlogTest.java | 1 - .../store/dledger/MessageStoreTestBase.java | 13 ++++++++++++ .../store/dledger/MixCommitlogTest.java | 20 ++++++++++++++++++- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 655c4de4c7..cb6c5f6565 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -234,6 +234,11 @@ public class DLedgerCommitLog extends CommitLog { dLedgerFileStore.load(); if (dLedgerFileList.getMappedFiles().size() > 0) { dLedgerFileStore.recover(); + MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); + if (mappedFile != null) { + dLedgerConfig.setEnableDiskForceClean(false); + dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); + } return; } //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog @@ -248,7 +253,6 @@ public class DLedgerCommitLog extends CommitLog { if (mappedFile == null) { return; } - dLedgerConfig.setEnableDiskForceClean(false); ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); byteBuffer.position(mappedFile.getWrotePosition()); boolean needWriteMagicCode = true; @@ -260,6 +264,7 @@ public class DLedgerCommitLog extends CommitLog { } else { log.info("Recover old commitlog found a illegal magic code={}", magicCode); } + dLedgerConfig.setEnableDiskForceClean(false); dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); log.info("Recover old commitlog needWriteMagicCode={} pos={} file={} dividedCommitlogOffset={}", needWriteMagicCode, mappedFile.getFileFromOffset() + mappedFile.getWrotePosition(), mappedFile.getFileName(), dividedCommitlogOffset); if (needWriteMagicCode) { @@ -465,7 +470,7 @@ public class DLedgerCommitLog extends CommitLog { @Override public SelectMappedBufferResult getMessage(final long offset, final int size) { if (offset < dividedCommitlogOffset) { - return getMessage(offset, size); + return super.getMessage(offset, size); } int mappedFileSize = this.dLedgerServer.getdLedgerConfig().getMappedFileSizeForEntryData(); MmapFile mappedFile = this.dLedgerFileList.findMappedFileByOffset(offset, offset == 0); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 28a1d8ea10..47d4b3874a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -23,7 +23,6 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); - { DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); Thread.sleep(1000); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index d671287d89..bdc9761eb1 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -3,7 +3,10 @@ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.DLedgerServer; import java.io.File; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; @@ -89,4 +92,14 @@ public class MessageStoreTestBase extends StoreTestBase { } } + protected void doGetMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) { + for (int i = 0; i < num; i++) { + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, beginLogicsOffset + i, 3, null); + Assert.assertNotNull(getMessageResult); + Assert.assertTrue(!getMessageResult.getMessageBufferList().isEmpty()); + MessageExt messageExt = MessageDecoder.decode(getMessageResult.getMessageBufferList().get(0)); + Assert.assertEquals(beginLogicsOffset + i, messageExt.getQueueOffset()); + } + } + } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index cba9fe9e0f..5ade1dfd62 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -5,7 +5,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.junit.Assert; import org.junit.Test; -public class MixCommitlogTest extends DLedgerCommitlogTest { +public class MixCommitlogTest extends MessageStoreTestBase { @Test @@ -25,6 +25,7 @@ public class MixCommitlogTest extends DLedgerCommitlogTest { Assert.assertEquals(0, originalStore.dispatchBehindBytes()); dividedOffset = originalStore.getCommitLog().getMaxOffset(); dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + doGetMessages(originalStore, topic, 0, 1000, 0); originalStore.shutdown(); } { @@ -33,11 +34,13 @@ public class MixCommitlogTest extends DLedgerCommitlogTest { Assert.assertEquals(0, recoverOriginalStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(1000, recoverOriginalStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, recoverOriginalStore.dispatchBehindBytes()); + doGetMessages(recoverOriginalStore, topic, 0, 1000, 0); recoverOriginalStore.shutdown(); } { DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); + Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); Thread.sleep(2000); doPutMessages(dledgerStore, topic, 0, 1000, 1000); @@ -45,8 +48,23 @@ public class MixCommitlogTest extends DLedgerCommitlogTest { Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + doGetMessages(dledgerStore, topic, 0, 2000, 0); dledgerStore.shutdown(); } + { + DefaultMessageStore recoverDledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) recoverDledgerStore.getCommitLog(); + Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); + Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); + Thread.sleep(2000); + doPutMessages(recoverDledgerStore, topic, 0, 1000, 2000); + Thread.sleep(500); + Assert.assertEquals(0, recoverDledgerStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(3000, recoverDledgerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, recoverDledgerStore.dispatchBehindBytes()); + doGetMessages(recoverDledgerStore, topic, 0, 3000, 0); + recoverDledgerStore.shutdown(); + } } } From 44f696c33d23cd15e912ebc0018ff79d101536c7 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 4 Dec 2018 19:59:40 +0800 Subject: [PATCH 032/122] Add deleting test for mixed commitlog --- .../store/dledger/DLedgerCommitLog.java | 29 ++++++++-- .../store/dledger/DLedgerCommitlogTest.java | 11 +++- .../store/dledger/MessageStoreTestBase.java | 1 + .../store/dledger/MixCommitlogTest.java | 58 +++++++++++++++++++ 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index cb6c5f6565..b5ebefe9d6 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -30,6 +30,7 @@ import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -70,6 +71,7 @@ public class DLedgerCommitLog extends CommitLog { //The old commitlog should be deleted before the dledger commitlog private final boolean originalDledgerEnableForceClean; + private final AtomicBoolean hasSetOriginalDledgerEnableForceClean = new AtomicBoolean(false); private boolean isInrecoveringOldCommitlog = false; @@ -171,12 +173,28 @@ public class DLedgerCommitLog extends CommitLog { final long intervalForcibly, final boolean cleanImmediately ) { - int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); if (mappedFileQueue.getMappedFiles().isEmpty()) { - dLedgerConfig.setEnableDiskForceClean(originalDledgerEnableForceClean); + if (hasSetOriginalDledgerEnableForceClean.compareAndSet(false, true)) { + dLedgerConfig.setEnableDiskForceClean(originalDledgerEnableForceClean); + } + //To prevent too much log in defaultMessageStore + return Integer.MAX_VALUE; } - //return 1 to prevent too much log in defaultMessageStore - return count > 0 ? count : 1; + int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); + if (count > 0 || mappedFileQueue.getMappedFiles().size() != 1) { + return count; + } + //the old logic will keep the last file, here to delete it + MappedFile mappedFile = mappedFileQueue.getLastMappedFile(); + log.info("Try to delete the last old commitlog file {}", mappedFile.getFileName()); + long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime; + if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) { + while (!mappedFile.destroy(10 * 1000)) { + io.openmessaging.storage.dledger.utils.UtilAll.sleep(1000); + } + mappedFileQueue.getMappedFiles().remove(mappedFile); + } + return 1; } @@ -238,6 +256,8 @@ public class DLedgerCommitLog extends CommitLog { if (mappedFile != null) { dLedgerConfig.setEnableDiskForceClean(false); dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); + } else { + hasSetOriginalDledgerEnableForceClean.set(true); } return; } @@ -251,6 +271,7 @@ public class DLedgerCommitLog extends CommitLog { isInrecoveringOldCommitlog = false; MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile == null) { + hasSetOriginalDledgerEnableForceClean.set(true); return; } ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 47d4b3874a..e32fabb7eb 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -19,33 +19,40 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { @Test - public void testReputOffset() throws Exception { + public void testRecover() throws Exception { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); + String topic = UUID.randomUUID().toString(); { DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); Thread.sleep(1000); - String topic = UUID.randomUUID().toString(); doPutMessages(messageStore, topic, 0, 1000, 0); Thread.sleep(100); Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + doGetMessages(messageStore, topic, 0, 1000, 0); messageStore.shutdown(); } { //normal recover DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + doGetMessages(messageStore, topic, 0, 1000, 0); messageStore.shutdown(); } { //abnormal recover DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + doGetMessages(messageStore, topic, 0, 1000, 0); messageStore.shutdown(); } } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index bdc9761eb1..fb7e3a1b58 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -99,6 +99,7 @@ public class MessageStoreTestBase extends StoreTestBase { Assert.assertTrue(!getMessageResult.getMessageBufferList().isEmpty()); MessageExt messageExt = MessageDecoder.decode(getMessageResult.getMessageBufferList().get(0)); Assert.assertEquals(beginLogicsOffset + i, messageExt.getQueueOffset()); + getMessageResult.release(); } } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index 5ade1dfd62..7bca5ff3cb 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -65,6 +65,64 @@ public class MixCommitlogTest extends MessageStoreTestBase { doGetMessages(recoverDledgerStore, topic, 0, 3000, 0); recoverDledgerStore.shutdown(); } + } + @Test + public void testDeleteExpiredFiles() throws Exception { + String base = createBaseDir(); + String topic = UUID.randomUUID().toString(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + + long dividedOffset; + { + DefaultMessageStore originalStore = createMessageStore(base, false); + doPutMessages(originalStore, topic, 0, 1000, 0); + Thread.sleep(500); + Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, originalStore.dispatchBehindBytes()); + dividedOffset = originalStore.getCommitLog().getMaxOffset(); + dividedOffset = dividedOffset - dividedOffset % originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog() + originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + originalStore.shutdown(); + } + long maxPhysicalOffset; + { + DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); + Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); + Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); + Thread.sleep(2000); + doPutMessages(dledgerStore, topic, 0, 1000, 1000); + Thread.sleep(500); + Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + Assert.assertEquals(0, dledgerStore.getMinPhyOffset()); + maxPhysicalOffset = dledgerStore.getMaxPhyOffset(); + + doGetMessages(dledgerStore, topic, 0, 2000, 0); + + for (int i = 0; i < 100; i++) { + dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true); + } + Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset()); + Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset()); + for (int i = 0; i < 100; i++) { + Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true)); + } + Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset()); + Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset()); + Assert.assertTrue(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); + + dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().setEnableDiskForceClean(false); + for (int i = 0; i < 100; i++) { + Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true)); + } + //should not change the value + Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); + doGetMessages(dledgerStore, topic, 0, 1000, 1000); + dledgerStore.shutdown(); + } } } From 30028a1104368c653a99aa52f2e2345865bf43db Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 5 Dec 2018 14:50:28 +0800 Subject: [PATCH 033/122] Make ServiceThread be able to be restarted --- .../apache/rocketmq/common/ServiceThread.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index 53ca329082..2417cf09dc 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -27,18 +27,27 @@ public abstract class ServiceThread implements Runnable { private static final long JOIN_TIME = 90 * 1000; - protected final Thread thread; + protected Thread thread; protected final CountDownLatch2 waitPoint = new CountDownLatch2(1); protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false); protected volatile boolean stopped = false; + //Make it able to restart the thread + private final AtomicBoolean started = new AtomicBoolean(false); + public ServiceThread() { - this.thread = new Thread(this, this.getServiceName()); + } public abstract String getServiceName(); public void start() { + log.info("Try to start service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread); + if (!started.compareAndSet(false, true)) { + return; + } + stopped = false; + this.thread = new Thread(this, getServiceName()); this.thread.start(); } @@ -47,6 +56,10 @@ public abstract class ServiceThread implements Runnable { } public void shutdown(final boolean interrupt) { + log.info("Try to start service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread); + if (!started.compareAndSet(true, false)) { + return; + } this.stopped = true; log.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt); @@ -80,6 +93,9 @@ public abstract class ServiceThread implements Runnable { } public void stop(final boolean interrupt) { + if (!started.get()) { + return; + } this.stopped = true; log.info("stop thread " + this.getServiceName() + " interrupt " + interrupt); @@ -93,6 +109,9 @@ public abstract class ServiceThread implements Runnable { } public void makeStop() { + if (!started.get()) { + return; + } this.stopped = true; log.info("makestop thread " + this.getServiceName()); } From 9c8fa5e16b33ef62483069f70d6eb12fce17e1e4 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 5 Dec 2018 14:51:19 +0800 Subject: [PATCH 034/122] Catch service thread exception of scheduledMessage and transactionalMessage when changing role --- .../rocketmq/broker/BrokerController.java | 24 +++++++++++++++---- .../dledger/DLedgerRoleChangeHandler.java | 7 ++++-- .../rocketmq/broker/out/BrokerOuterAPI.java | 2 +- .../TransactionalMessageCheckService.java | 21 ---------------- 4 files changed, 26 insertions(+), 28 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 4b3c0a4683..acdf52aba2 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -1067,10 +1067,18 @@ public class BrokerController { messageStoreConfig.setBrokerRole(BrokerRole.SLAVE); //handle the scheduled service - this.messageStore.handleScheduleMessageService(BrokerRole.SLAVE); + try { + this.messageStore.handleScheduleMessageService(BrokerRole.SLAVE); + } catch (Throwable t) { + log.error("[MONITOR] handleScheduleMessageService failed when changing to slave", t); + } //handle the transactional service - this.shutdownProcessorByHa(); + try { + this.shutdownProcessorByHa(); + } catch (Throwable t) { + log.error("[MONITOR] shutdownProcessorByHa failed when changing to slave", t); + } //handle the slave synchronise handleSlaveSynchronize(BrokerRole.SLAVE); @@ -1095,10 +1103,18 @@ public class BrokerController { handleSlaveSynchronize(role); //handle the scheduled service - this.messageStore.handleScheduleMessageService(role); + try { + this.messageStore.handleScheduleMessageService(role); + } catch (Throwable t) { + log.error("[MONITOR] handleScheduleMessageService failed when changing to master", t); + } //handle the transactional service - this.startProcessorByHa(BrokerRole.SYNC_MASTER); + try { + this.startProcessorByHa(BrokerRole.SYNC_MASTER); + } catch (Throwable t) { + log.error("[MONITOR] startProcessorByHa failed when changing to master", t); + } //if the operations above are totally successful, we change to master brokerConfig.setBrokerId(0); //TO DO check diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index b1f3bdcb22..88f83ff3ab 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -51,33 +51,36 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange @Override public void run() { long start = System.currentTimeMillis(); try { + boolean succ = false; log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); switch (role) { case CANDIDATE: if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { brokerController.changeToSlave(dLedgerCommitLog.getId()); } + succ = true; break; case FOLLOWER: brokerController.changeToSlave(dLedgerCommitLog.getId()); + succ = true; break; case LEADER: while (dLegerServer.getMemberState().isLeader() && (dLegerServer.getdLedgerStore().getLedgerEndIndex() != dLegerServer.getdLedgerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { UtilAll.sleep(100); } - boolean succ = dLegerServer.getMemberState().isLeader() + succ = dLegerServer.getMemberState().isLeader() && dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex() && messageStore.dispatchBehindBytes() == 0; if (succ) { messageStore.recoverTopicQueueTable(); brokerController.changeToMaster(BrokerRole.SYNC_MASTER); } - log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); break; default: break; } + log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); } catch (Throwable t) { log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 4dee01cbff..6caa2358e3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -152,7 +152,7 @@ public class BrokerOuterAPI { registerBrokerResultList.add(result); } - log.info("register broker to name server {} OK", namesrvAddr); + log.info("register broker[{}]to name server {} OK", brokerId, namesrvAddr); } catch (Exception e) { log.warn("registerBroker Exception, {}", namesrvAddr, e); } finally { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index bd98d0e78b..d85283d04d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -29,31 +29,10 @@ public class TransactionalMessageCheckService extends ServiceThread { private BrokerController brokerController; - private final AtomicBoolean started = new AtomicBoolean(false); - public TransactionalMessageCheckService(BrokerController brokerController) { this.brokerController = brokerController; } - @Override - public void start() { - if (started.compareAndSet(false, true)) { - super.start(); - //no need to do this - //this.brokerController.getTransactionalMessageService().open(); - } - } - - @Override - public void shutdown(boolean interrupt) { - if (started.compareAndSet(true, false)) { - super.shutdown(interrupt); - //no need to do this - //this.brokerController.getTransactionalMessageService().close(); - //this.brokerController.getTransactionalMessageCheckListener().shutDown(); - } - } - @Override public String getServiceName() { return TransactionalMessageCheckService.class.getSimpleName(); From 235ac796292471bb97218626b003e0b0173ff312 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 5 Dec 2018 14:53:44 +0800 Subject: [PATCH 035/122] Polish checkstyle --- .../broker/transaction/TransactionalMessageCheckService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index d85283d04d..a2cc0baa9e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -22,8 +22,6 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; -import java.util.concurrent.atomic.AtomicBoolean; - public class TransactionalMessageCheckService extends ServiceThread { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TRANSACTION_LOGGER_NAME); From 415b38bdc92d6326ac9b0814610207926237c195 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 11 Dec 2018 16:15:22 +0800 Subject: [PATCH 036/122] Do not return flush_slave_timeout to client --- .../rocketmq/store/dledger/DLedgerCommitLog.java | 12 +++++++----- .../rocketmq/store/dledger/DLedgerCommitlogTest.java | 2 +- .../rocketmq/store/dledger/MessageStoreTestBase.java | 2 ++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index b5ebefe9d6..86105ca4d2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -27,6 +27,7 @@ import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; import io.openmessaging.storage.dledger.store.file.MmapFile; import io.openmessaging.storage.dledger.store.file.MmapFileList; import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; +import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.concurrent.TimeUnit; @@ -190,7 +191,7 @@ public class DLedgerCommitLog extends CommitLog { long liveMaxTimestamp = mappedFile.getLastModifiedTimestamp() + expiredTime; if (System.currentTimeMillis() >= liveMaxTimestamp || cleanImmediately) { while (!mappedFile.destroy(10 * 1000)) { - io.openmessaging.storage.dledger.utils.UtilAll.sleep(1000); + DLedgerUtils.sleep(1000); } mappedFileQueue.getMappedFiles().remove(mappedFile); } @@ -397,7 +398,6 @@ public class DLedgerCommitLog extends CommitLog { } // Back to Results - PutMessageStatus putMessageStatus = null; AppendMessageResult appendResult; AppendFuture dledgerFuture; EncodeResult encodeResult; @@ -449,6 +449,7 @@ public class DLedgerCommitLog extends CommitLog { log.warn("[NOTIFYME]putMessage in lock cost time(ms)={}, bodyLength={} AppendMessageResult={}", eclipseTimeInLock, msg.getBody().length, appendResult); } + PutMessageStatus putMessageStatus = PutMessageStatus.UNKNOWN_ERROR; try { AppendEntryResponse appendEntryResponse = dledgerFuture.get(3, TimeUnit.SECONDS); switch (DLedgerResponseCode.valueOf(appendEntryResponse.getCode())) { @@ -462,14 +463,15 @@ public class DLedgerCommitLog extends CommitLog { putMessageStatus = PutMessageStatus.SERVICE_NOT_AVAILABLE; break; case WAIT_QUORUM_ACK_TIMEOUT: - putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; + //Do not return flush_slave_timeout to the client, for the ons client will ignore it. + putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; break; case LEADER_PENDING_FULL: putMessageStatus = PutMessageStatus.OS_PAGECACHE_BUSY; break; } - } catch (Throwable ignored) { - putMessageStatus = PutMessageStatus.FLUSH_SLAVE_TIMEOUT; + } catch (Throwable t) { + log.error("Failed to get dledger append result", t); } PutMessageResult putMessageResult = new PutMessageResult(putMessageStatus, appendResult); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index e32fabb7eb..1c774b91de 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -111,7 +111,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { msgInner.setTopic(topic); msgInner.setQueueId(0); PutMessageResult putMessageResult = leaderStore.putMessage(msgInner); - Assert.assertEquals(PutMessageStatus.FLUSH_SLAVE_TIMEOUT, putMessageResult.getPutMessageStatus()); + Assert.assertEquals(PutMessageStatus.OS_PAGECACHE_BUSY, putMessageResult.getPutMessageStatus()); Thread.sleep(1000); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index fb7e3a1b58..8cb1eff3ba 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -21,6 +21,8 @@ import org.junit.Assert; public class MessageStoreTestBase extends StoreTestBase { protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort) throws Exception { + System.setProperty("dledger.disk.ratio.check", "0.95"); + System.setProperty("dledger.disk.ratio.clean", "0.95"); baseDirs.add(base); MessageStoreConfig storeConfig = new MessageStoreConfig(); storeConfig.setMapedFileSizeCommitLog(1024 * 100); From f8dd3f70f2098a4ca95a5281313d1b495120c2ba Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 11 Dec 2018 16:20:07 +0800 Subject: [PATCH 037/122] Fix import names --- .../broker/dledger/DLedgerRoleChangeHandler.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index 88f83ff3ab..b9d00abc59 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -16,15 +16,15 @@ */ package org.apache.rocketmq.broker.dledger; +import io.openmessaging.storage.dledger.DLedgerLeaderElector; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.MemberState; +import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; -import io.openmessaging.storage.dledger.DLedgerLeaderElector; -import io.openmessaging.storage.dledger.DLedgerServer; -import io.openmessaging.storage.dledger.MemberState; -import io.openmessaging.storage.dledger.utils.UtilAll; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; @@ -67,7 +67,7 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange case LEADER: while (dLegerServer.getMemberState().isLeader() && (dLegerServer.getdLedgerStore().getLedgerEndIndex() != dLegerServer.getdLedgerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { - UtilAll.sleep(100); + DLedgerUtils.sleep(100); } succ = dLegerServer.getMemberState().isLeader() && dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex() @@ -80,9 +80,9 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange default: break; } - log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start)); + log.info("Finish handling broker role change succ={} term={} role={} currStoreRole={} cost={}", succ, term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start)); } catch (Throwable t) { - log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), UtilAll.elapsed(start), t); + log.info("[MONITOR]Failed handling broker role change term={} role={} currStoreRole={} cost={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole(), DLedgerUtils.elapsed(start), t); } } }; From 1276539ea7b49e018a655d5a2d56ed0bdd7ff491 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 13 Dec 2018 15:24:34 +0800 Subject: [PATCH 038/122] Add test for trucating consumequeue if the dledger commitlog is deleted --- .../apache/rocketmq/store/ConsumeQueue.java | 1 + .../rocketmq/store/MappedFileQueue.java | 4 +- .../store/dledger/DLedgerCommitLog.java | 8 ++- .../store/dledger/DLedgerCommitlogTest.java | 72 +++++++++++++++++-- .../store/dledger/MessageStoreTestBase.java | 23 +++++- .../store/dledger/MixCommitlogTest.java | 6 +- 6 files changed, 100 insertions(+), 14 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index af8445275b..a63d3746b3 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -421,6 +421,7 @@ public class ConsumeQueue { final long cqOffset) { if (offset + size <= this.maxPhysicOffset) { + log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); return true; } diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java index 86de3d2c1d..cc145921ce 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java @@ -154,8 +154,8 @@ public class MappedFileQueue { if (file.length() != this.mappedFileSize) { log.warn(file + "\t" + file.length() - + " length not matched message store config value, ignore it"); - return true; + + " length not matched message store config value, please check it manually"); + return false; } try { diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 86105ca4d2..88ad1ff912 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -253,13 +253,19 @@ public class DLedgerCommitLog extends CommitLog { dLedgerFileStore.load(); if (dLedgerFileList.getMappedFiles().size() > 0) { dLedgerFileStore.recover(); + dividedCommitlogOffset = dLedgerFileList.getFirstMappedFile().getFileFromOffset(); MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile != null) { dLedgerConfig.setEnableDiskForceClean(false); - dividedCommitlogOffset = mappedFile.getFileFromOffset() + mappedFile.getFileSize(); } else { hasSetOriginalDledgerEnableForceClean.set(true); } + long maxPhyOffset = dLedgerFileList.getMaxWrotePosition(); + // Clear ConsumeQueue redundant data + if (maxPhyOffsetOfConsumeQueue >= maxPhyOffset) { + log.warn("[TruncateCQ]maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, maxPhyOffset); + this.defaultMessageStore.truncateDirtyLogicFiles(maxPhyOffset); + } return; } //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 1c774b91de..70887d03fe 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -1,5 +1,8 @@ package org.apache.rocketmq.store.dledger; +import io.openmessaging.storage.dledger.DLedgerServer; +import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +import io.openmessaging.storage.dledger.store.file.MmapFileList; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -18,6 +21,63 @@ import org.junit.Test; public class DLedgerCommitlogTest extends MessageStoreTestBase { + @Test + public void testTruncateCQ() throws Exception { + String base = createBaseDir(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + String topic = UUID.randomUUID().toString(); + { + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); + DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer(); + DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); + MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList(); + Thread.sleep(2000); + doPutMessages(messageStore, topic, 0, 2000, 0); + Thread.sleep(100); + Assert.assertEquals(24, mmapFileList.getMappedFiles().size()); + System.out.println(mmapFileList.getMappedFiles().size()); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(2000, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + doGetMessages(messageStore, topic, 0, 2000, 0); + messageStore.shutdown(); + } + + { + //Abnormal recover, left some commitlogs + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 4); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); + DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer(); + DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); + MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList(); + Thread.sleep(1000); + Assert.assertEquals(20, mmapFileList.getMappedFiles().size()); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1700, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + doGetMessages(messageStore, topic, 0, 1700, 0); + messageStore.shutdown(); + } + { + //Abnormal recover, left none commitlogs + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 20); + DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); + DLedgerServer dLedgerServer = dLedgerCommitLog.getdLedgerServer(); + DLedgerMmapFileStore dLedgerMmapFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); + MmapFileList mmapFileList = dLedgerMmapFileStore.getDataFileList(); + Thread.sleep(1000); + Assert.assertEquals(0, mmapFileList.getMappedFiles().size()); + Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, messageStore.dispatchBehindBytes()); + messageStore.shutdown(); + } + } + + + @Test public void testRecover() throws Exception { String base = createBaseDir(); @@ -25,7 +85,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { String group = UUID.randomUUID().toString(); String topic = UUID.randomUUID().toString(); { - DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); Thread.sleep(1000); doPutMessages(messageStore, topic, 0, 1000, 0); Thread.sleep(100); @@ -38,7 +98,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { { //normal recover - DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); @@ -48,7 +108,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { { //abnormal recover - DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(1000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); @@ -64,7 +124,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { String base = createBaseDir(); String peers = String.format("n0-localhost:%d", nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false); + DefaultMessageStore messageStore = createDledgerMessageStore(base, group, "n0", peers, null, false, 0); Thread.sleep(1000); String topic = UUID.randomUUID().toString(); @@ -104,7 +164,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { public void testCommittedPos() throws Exception { String peers = String.format("n0-localhost:%d;n1-localhost:%d", nextPort(), nextPort()); String group = UUID.randomUUID().toString(); - DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false); + DefaultMessageStore leaderStore = createDledgerMessageStore(createBaseDir(), group,"n0", peers, "n0", false, 0); String topic = UUID.randomUUID().toString(); MessageExtBrokerInner msgInner = buildMessage(); @@ -119,7 +179,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); - DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false); + DefaultMessageStore followerStore = createDledgerMessageStore(createBaseDir(), group,"n1", peers, "n0", false, 0); Thread.sleep(2000); Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index 8cb1eff3ba..fb9a7798df 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -1,7 +1,9 @@ package org.apache.rocketmq.store.dledger; +import io.openmessaging.storage.dledger.DLedgerConfig; import io.openmessaging.storage.dledger.DLedgerServer; import java.io.File; +import java.util.Arrays; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -20,7 +22,7 @@ import org.junit.Assert; public class MessageStoreTestBase extends StoreTestBase { - protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort) throws Exception { + protected DefaultMessageStore createDledgerMessageStore(String base, String group, String selfId, String peers, String leaderId, boolean createAbort, int deleteFileNum) throws Exception { System.setProperty("dledger.disk.ratio.check", "0.95"); System.setProperty("dledger.disk.ratio.clean", "0.95"); baseDirs.add(base); @@ -40,8 +42,8 @@ public class MessageStoreTestBase extends StoreTestBase { DefaultMessageStore defaultMessageStore = new DefaultMessageStore(storeConfig, new BrokerStatsManager("DLedgerCommitlogTest"), (topic, queueId, logicOffset, tagsCode, msgStoreTime, filterBitMap, properties) -> { }, new BrokerConfig()); + DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); if (leaderId != null) { - DLedgerServer dLegerServer = ((DLedgerCommitLog) defaultMessageStore.getCommitLog()).getdLedgerServer(); dLegerServer.getdLedgerConfig().setEnableLeaderElector(false); if (selfId.equals(leaderId)) { dLegerServer.getMemberState().changeToLeader(-1); @@ -54,6 +56,23 @@ public class MessageStoreTestBase extends StoreTestBase { String fileName = StorePathConfigHelper.getAbortFile(storeConfig.getStorePathRootDir()); makeSureFileExists(fileName); } + if (deleteFileNum > 0) { + DLedgerConfig config = dLegerServer.getdLedgerConfig(); + if (deleteFileNum > 0) { + File dir = new File(config.getDataStorePath()); + File[] files = dir.listFiles(); + if (files != null) { + Arrays.sort(files); + for (int i = files.length - 1; i >= 0; i--) { + File file = files[i]; + file.delete(); + if (files.length - i >= deleteFileNum) { + break; + } + } + } + } + } Assert.assertTrue(defaultMessageStore.load()); defaultMessageStore.start(); return defaultMessageStore; diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index 7bca5ff3cb..690348e836 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -38,7 +38,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { recoverOriginalStore.shutdown(); } { - DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); @@ -52,7 +52,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { dledgerStore.shutdown(); } { - DefaultMessageStore recoverDledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DefaultMessageStore recoverDledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) recoverDledgerStore.getCommitLog(); Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); @@ -88,7 +88,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { } long maxPhysicalOffset; { - DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true); + DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); From 698ef2f3d9f0f62ddccf905e90d1497630c054df Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 13 Dec 2018 17:52:24 +0800 Subject: [PATCH 039/122] Fix old ha service --- .../rocketmq/broker/BrokerController.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index acdf52aba2..32655ce405 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -400,6 +400,29 @@ public class BrokerController { } }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); } + + if (!messageStoreConfig.isEnableDLegerCommitLog()) { + if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { + if (this.messageStoreConfig.getHaMasterAddress() != null && this.messageStoreConfig.getHaMasterAddress().length() >= 6) { + this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress()); + this.updateMasterHAServerAddrPeriodically = false; + } else { + this.updateMasterHAServerAddrPeriodically = true; + } + } else { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.printMasterAndSlaveDiff(); + } catch (Throwable e) { + log.error("schedule printMasterAndSlaveDiff error.", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } + } + if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { // Register a listener to reload SslContext try { From 38554a089c3d800d38d8dd03321b066389bccce3 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 13 Dec 2018 20:54:51 +0800 Subject: [PATCH 040/122] Polish ServiceThread to make inner thread private --- .../apache/rocketmq/common/ServiceThread.java | 20 +++++++++++++++---- .../store/AllocateMappedFileService.java | 11 ++-------- .../rocketmq/store/ha/HAConnection.java | 4 ++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index 2417cf09dc..0e33418f1b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -27,10 +27,11 @@ public abstract class ServiceThread implements Runnable { private static final long JOIN_TIME = 90 * 1000; - protected Thread thread; + private Thread thread; protected final CountDownLatch2 waitPoint = new CountDownLatch2(1); protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false); protected volatile boolean stopped = false; + protected boolean isDaemon = false; //Make it able to restart the thread private final AtomicBoolean started = new AtomicBoolean(false); @@ -48,6 +49,7 @@ public abstract class ServiceThread implements Runnable { } stopped = false; this.thread = new Thread(this, getServiceName()); + this.thread.setDaemon(isDaemon); this.thread.start(); } @@ -56,7 +58,7 @@ public abstract class ServiceThread implements Runnable { } public void shutdown(final boolean interrupt) { - log.info("Try to start service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread); + log.info("Try to shutdown service thread:{} started:{} lastThread:{}", getServiceName(), started.get(), thread); if (!started.compareAndSet(true, false)) { return; } @@ -88,12 +90,14 @@ public abstract class ServiceThread implements Runnable { return JOIN_TIME; } + @Deprecated public void stop() { this.stop(false); } + @Deprecated public void stop(final boolean interrupt) { - if (!started.get()) { + if (!started.compareAndSet(true, false)) { return; } this.stopped = true; @@ -109,7 +113,7 @@ public abstract class ServiceThread implements Runnable { } public void makeStop() { - if (!started.get()) { + if (!started.compareAndSet(true, false)) { return; } this.stopped = true; @@ -147,4 +151,12 @@ public abstract class ServiceThread implements Runnable { public boolean isStopped() { return stopped; } + + public boolean isDaemon() { + return isDaemon; + } + + public void setDaemon(boolean daemon) { + isDaemon = daemon; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java index dbb27cd744..5e86b8a85f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java @@ -120,16 +120,9 @@ public class AllocateMappedFileService extends ServiceThread { return AllocateMappedFileService.class.getSimpleName(); } + @Override public void shutdown() { - this.stopped = true; - this.thread.interrupt(); - - try { - this.thread.join(this.getJointime()); - } catch (InterruptedException e) { - log.error("Interrupted", e); - } - + super.shutdown(true); for (AllocateRequest req : this.requestTable.values()) { if (req.mappedFile != null) { log.info("delete pre allocated maped file, {}", req.mappedFile.getFileName()); diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java index 251dc9971f..c102881c1c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAConnection.java @@ -90,7 +90,7 @@ public class HAConnection { this.selector = RemotingUtil.openSelector(); this.socketChannel = socketChannel; this.socketChannel.register(this.selector, SelectionKey.OP_READ); - this.thread.setDaemon(true); + this.setDaemon(true); } @Override @@ -205,7 +205,7 @@ public class HAConnection { this.selector = RemotingUtil.openSelector(); this.socketChannel = socketChannel; this.socketChannel.register(this.selector, SelectionKey.OP_WRITE); - this.thread.setDaemon(true); + this.setDaemon(true); } @Override From 43753d6e3e10b62d8012cc24425b6eab2707e7ce Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 13 Dec 2018 21:05:01 +0800 Subject: [PATCH 041/122] Polish makeStop for ServiceThread --- .../main/java/org/apache/rocketmq/common/ServiceThread.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index 0e33418f1b..ae46e78fa6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -97,7 +97,7 @@ public abstract class ServiceThread implements Runnable { @Deprecated public void stop(final boolean interrupt) { - if (!started.compareAndSet(true, false)) { + if (!started.get()) { return; } this.stopped = true; @@ -113,7 +113,7 @@ public abstract class ServiceThread implements Runnable { } public void makeStop() { - if (!started.compareAndSet(true, false)) { + if (!started.get()) { return; } this.stopped = true; From 1e358ae359763304a24c282eecc102d88ced75be Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 17 Dec 2018 10:44:57 +0800 Subject: [PATCH 042/122] Annotate --- .../org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 88ad1ff912..8c93e7df99 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -270,6 +270,8 @@ public class DLedgerCommitLog extends CommitLog { } //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog isInrecoveringOldCommitlog = true; + //TODO only do the normal recover + //No need the abnormal recover if (lastOk) { super.recoverNormally(maxPhyOffsetOfConsumeQueue); } else { @@ -300,6 +302,7 @@ public class DLedgerCommitLog extends CommitLog { byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition()); byteBuffer.putInt(BLANK_MAGIC_CODE); mappedFile.flush(0); + //TODO already set mappedFile.setWrotePosition(mappedFile.getFileSize()); mappedFile.setCommittedPosition(mappedFile.getFileSize()); mappedFile.setFlushedPosition(mappedFile.getFileSize()); From a50b733df8bc4e3d4df94eff7391c5c5de60c95d Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 17 Dec 2018 16:55:25 +0800 Subject: [PATCH 043/122] Add tests for consume queue fall behind --- .../org/apache/rocketmq/store/CommitLog.java | 7 +++ .../rocketmq/store/DefaultMessageStore.java | 9 +++- .../store/dledger/DLedgerCommitLog.java | 14 ++---- .../store/dledger/DLedgerCommitlogTest.java | 1 - .../store/dledger/MixCommitlogTest.java | 43 +++++++++++++++++++ 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 64437565e2..cbcc1a7b72 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -212,6 +212,12 @@ public class CommitLog { log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); } + } else { + // Commitlog case files are deleted + log.warn("The commitlog files are deleted, and delete the consume queue files"); + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); + this.defaultMessageStore.destroyLogics(); } } @@ -396,6 +402,7 @@ public class CommitLog { this.confirmOffset = phyOffset; } + @Deprecated public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { // recover by the minimum time stamp boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 91d51530ba..cdbd1a8bd8 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -236,7 +236,6 @@ public class DefaultMessageStore implements MessageStore { } } } - log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); long reputOffset; @@ -256,6 +255,13 @@ public class DefaultMessageStore implements MessageStore { this.handleScheduleMessageService(messageStoreConfig.getBrokerRole()); } + //Finish dispatching the messages fall behind + while (this.dispatchBehindBytes() != 0) { + Thread.sleep(1000); + log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes()); + } + this.recoverTopicQueueTable(); + this.createTempFile(); this.addScheduleTask(); this.shutdown = false; @@ -1064,6 +1070,7 @@ public class DefaultMessageStore implements MessageStore { return false; } + @Override public long dispatchBehindBytes() { return this.reputMessageService.behind(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 8c93e7df99..30ecefff3c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -270,13 +270,8 @@ public class DLedgerCommitLog extends CommitLog { } //Indicate that, it is the first time to load mixed commitlog, need to recover the old commitlog isInrecoveringOldCommitlog = true; - //TODO only do the normal recover //No need the abnormal recover - if (lastOk) { - super.recoverNormally(maxPhyOffsetOfConsumeQueue); - } else { - super.recoverAbnormally(maxPhyOffsetOfConsumeQueue); - } + super.recoverNormally(maxPhyOffsetOfConsumeQueue); isInrecoveringOldCommitlog = false; MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile == null) { @@ -302,11 +297,10 @@ public class DLedgerCommitLog extends CommitLog { byteBuffer.putInt(mappedFile.getFileSize() - mappedFile.getWrotePosition()); byteBuffer.putInt(BLANK_MAGIC_CODE); mappedFile.flush(0); - //TODO already set - mappedFile.setWrotePosition(mappedFile.getFileSize()); - mappedFile.setCommittedPosition(mappedFile.getFileSize()); - mappedFile.setFlushedPosition(mappedFile.getFileSize()); } + mappedFile.setWrotePosition(mappedFile.getFileSize()); + mappedFile.setCommittedPosition(mappedFile.getFileSize()); + mappedFile.setFlushedPosition(mappedFile.getFileSize()); dLedgerFileList.getLastMappedFile(dividedCommitlogOffset); log.info("Will set the initial commitlog offset={} for dledger", dividedCommitlogOffset); } diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 70887d03fe..5c098d4b9a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -37,7 +37,6 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { doPutMessages(messageStore, topic, 0, 2000, 0); Thread.sleep(100); Assert.assertEquals(24, mmapFileList.getMappedFiles().size()); - System.out.println(mmapFileList.getMappedFiles().size()); Assert.assertEquals(0, messageStore.getMinOffsetInQueue(topic, 0)); Assert.assertEquals(2000, messageStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(0, messageStore.dispatchBehindBytes()); diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index 690348e836..8fb62f3b23 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -2,12 +2,55 @@ package org.apache.rocketmq.store.dledger; import java.util.UUID; import org.apache.rocketmq.store.DefaultMessageStore; +import org.apache.rocketmq.store.StoreTestBase; +import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.junit.Assert; import org.junit.Test; public class MixCommitlogTest extends MessageStoreTestBase { + + @Test + public void testFallBehindCQ() throws Exception { + String base = createBaseDir(); + String topic = UUID.randomUUID().toString(); + String peers = String.format("n0-localhost:%d", nextPort()); + String group = UUID.randomUUID().toString(); + { + DefaultMessageStore originalStore = createMessageStore(base, false); + doPutMessages(originalStore, topic, 0, 1000, 0); + Assert.assertEquals(11, originalStore.getMaxPhyOffset()/originalStore.getMessageStoreConfig().getMapedFileSizeCommitLog()); + Thread.sleep(500); + Assert.assertEquals(0, originalStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, originalStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, originalStore.dispatchBehindBytes()); + doGetMessages(originalStore, topic, 0, 1000, 0); + originalStore.shutdown(); + } + //delete the cq files + { + StoreTestBase.deleteFile(StorePathConfigHelper.getStorePathConsumeQueue(base)); + } + { + DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); + Thread.sleep(2000); + Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(1000, dledgerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + doGetMessages(dledgerStore, topic, 0, 1000, 0); + doPutMessages(dledgerStore, topic, 0, 1000, 1000); + Thread.sleep(500); + Assert.assertEquals(0, dledgerStore.getMinOffsetInQueue(topic, 0)); + Assert.assertEquals(2000, dledgerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + doGetMessages(dledgerStore, topic, 0, 2000, 0); + dledgerStore.shutdown(); + } + } + + + @Test public void testPutAndGet() throws Exception { String base = createBaseDir(); From a90342e9de4081db314485b6100299e03000c533 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 17 Dec 2018 17:32:56 +0800 Subject: [PATCH 044/122] No need to change to slave --- .../java/org/apache/rocketmq/broker/BrokerController.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 32655ce405..3418e82094 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -806,9 +806,7 @@ public class BrokerController { this.filterServerManager.start(); } - if (messageStoreConfig.isEnableDLegerCommitLog()) { - changeToSlave(((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getId()); - } else { + if (!messageStoreConfig.isEnableDLegerCommitLog()) { startProcessorByHa(messageStoreConfig.getBrokerRole()); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); } From 5ce20e8a37e5361bdf22124bbbf86f6ef305c2db Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 20 Dec 2018 15:14:37 +0800 Subject: [PATCH 045/122] Make sure the fallen behind node not to be leader --- .../dledger/DLedgerRoleChangeHandler.java | 23 ++++--- .../rocketmq/store/DefaultMessageStore.java | 65 ++++++++++--------- .../store/dledger/DLedgerCommitLog.java | 20 +++--- .../store/dledger/DLedgerCommitlogTest.java | 3 +- ...t.java => DLedgerProduceAndConsumeIT.java} | 0 5 files changed, 62 insertions(+), 49 deletions(-) rename test/src/test/java/org/apache/rocketmq/test/base/dledger/{ProduceAndConsumeTest.java => DLedgerProduceAndConsumeIT.java} (100%) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java index b9d00abc59..09bf10c327 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java @@ -51,27 +51,32 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange @Override public void run() { long start = System.currentTimeMillis(); try { - boolean succ = false; + boolean succ = true; log.info("Begin handling broker role change term={} role={} currStoreRole={}", term, role, messageStore.getMessageStoreConfig().getBrokerRole()); switch (role) { case CANDIDATE: if (messageStore.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE) { brokerController.changeToSlave(dLedgerCommitLog.getId()); } - succ = true; break; case FOLLOWER: brokerController.changeToSlave(dLedgerCommitLog.getId()); - succ = true; break; case LEADER: - while (dLegerServer.getMemberState().isLeader() - && (dLegerServer.getdLedgerStore().getLedgerEndIndex() != dLegerServer.getdLedgerStore().getCommittedIndex() || messageStore.dispatchBehindBytes() != 0)) { - DLedgerUtils.sleep(100); + while (true) { + if (!dLegerServer.getMemberState().isLeader()) { + succ = false; + break; + } + if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == -1) { + break; + } + if (dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex() + && messageStore.dispatchBehindBytes() == 0) { + break; + } + Thread.sleep(100); } - succ = dLegerServer.getMemberState().isLeader() - && dLegerServer.getdLedgerStore().getLedgerEndIndex() == dLegerServer.getdLedgerStore().getCommittedIndex() - && messageStore.dispatchBehindBytes() == 0; if (succ) { messageStore.recoverTopicQueueTable(); brokerController.changeToMaster(BrokerRole.SYNC_MASTER); diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index cdbd1a8bd8..124e14e780 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -222,45 +222,50 @@ public class DefaultMessageStore implements MessageStore { lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes())); lockFile.getChannel().force(true); - - this.flushConsumeQueueService.start(); - this.commitLog.start(); - this.storeStatsService.start(); - - //calculate the reput offset from the consume queue itself - long maxPhysicalPosInLogicQueue = commitLog.getMinOffset(); - for (ConcurrentMap maps : this.consumeQueueTable.values()) { - for (ConsumeQueue logic : maps.values()) { - if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) { - maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset(); + { + /** + * 1. calculate the reput offset according to the consume queue; + * 2. make sure the lagged messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed. + */ + long maxPhysicalPosInLogicQueue = commitLog.getMinOffset(); + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + if (logic.getMaxPhysicOffset() > maxPhysicalPosInLogicQueue) { + maxPhysicalPosInLogicQueue = logic.getMaxPhysicOffset(); + } } } + log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", + maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); + long reputOffset; + if (this.getMessageStoreConfig().isDuplicationEnable()) { + reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset()); + } else { + reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset()); + } + if (reputOffset < 0) { + reputOffset = 0; + } + this.reputMessageService.setReputFromOffset(reputOffset); + this.reputMessageService.start(); + + //Finish dispatching the messages fall behind + //Note that, if dledger is enabled, the maxOffset maybe -1, so here only require the dispatchBehindBytes > 0 + while (this.dispatchBehindBytes() > 0) { + Thread.sleep(1000); + log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes()); + } + this.recoverTopicQueueTable(); } - log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", - maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); - long reputOffset; - if (this.getMessageStoreConfig().isDuplicationEnable()) { - reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset()); - } else { - reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset()); - } - if (reputOffset < 0) { - reputOffset = 0; - } - this.reputMessageService.setReputFromOffset(reputOffset); - this.reputMessageService.start(); if (!messageStoreConfig.isEnableDLegerCommitLog()) { this.haService.start(); this.handleScheduleMessageService(messageStoreConfig.getBrokerRole()); } - //Finish dispatching the messages fall behind - while (this.dispatchBehindBytes() != 0) { - Thread.sleep(1000); - log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes()); - } - this.recoverTopicQueueTable(); + this.flushConsumeQueueService.start(); + this.commitLog.start(); + this.storeStatsService.start(); this.createTempFile(); this.addScheduleTask(); diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 30ecefff3c..831f778310 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -86,6 +86,7 @@ public class DLedgerCommitLog extends CommitLog { dLedgerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); dLedgerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog()); + dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen()); originalDledgerEnableForceClean = dLedgerConfig.isEnableDiskForceClean(); id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1; dLedgerServer = new DLedgerServer(dLedgerConfig); @@ -129,25 +130,26 @@ public class DLedgerCommitLog extends CommitLog { @Override public long getMaxOffset() { - if (this.dLedgerFileStore.getCommittedPos() != -1) { - return dLedgerFileStore.getCommittedPos(); - } else { - return this.dLedgerFileList.getMaxWrotePosition(); + if (dividedCommitlogOffset > 0 && dLedgerFileStore.getCommittedPos() < 0) { + return dividedCommitlogOffset; } + if (dLedgerFileStore.getCommittedPos() == -1) { + return 0; + } + return dLedgerFileStore.getCommittedPos(); } @Override public long getMinOffset() { - if (mappedFileQueue.getMappedFiles().isEmpty()) { - return dLedgerFileList.getMinOffset(); + if (!mappedFileQueue.getMappedFiles().isEmpty()) { + return mappedFileQueue.getMinOffset(); } - return mappedFileQueue.getMinOffset(); + return dLedgerFileList.getMinOffset(); } @Override public long getConfirmOffset() { - return this.dLedgerFileStore.getCommittedPos() == -1 ? getMaxOffset() - : this.dLedgerFileStore.getCommittedPos(); + return this.getMaxOffset(); } @Override diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index 5c098d4b9a..a6acbdb8d9 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -174,7 +174,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Thread.sleep(1000); - Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0); + Assert.assertEquals(0, leaderStore.getCommitLog().getMaxOffset()); Assert.assertEquals(0, leaderStore.getMaxOffsetInQueue(topic, 0)); @@ -183,6 +183,7 @@ public class DLedgerCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(1, leaderStore.getMaxOffsetInQueue(topic, 0)); Assert.assertEquals(1, followerStore.getMaxOffsetInQueue(topic, 0)); + Assert.assertTrue(leaderStore.getCommitLog().getMaxOffset() > 0); leaderStore.destroy(); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dledger/ProduceAndConsumeTest.java b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java similarity index 100% rename from test/src/test/java/org/apache/rocketmq/test/base/dledger/ProduceAndConsumeTest.java rename to test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java From 5bf113a17c1f9bba5cfdd3ae550991f561cbe18d Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 20 Dec 2018 15:15:13 +0800 Subject: [PATCH 046/122] Polish tests, separate ports, disable log, etc. --- README.md | 5 ++++- .../rocketmq/broker/processor/ClientManageProcessor.java | 2 +- .../rocketmq/client/impl/factory/MQClientInstance.java | 4 ++-- .../rocketmq/store/DefaultMessageStoreShutDownTest.java | 2 +- .../rocketmq/store/dledger/MessageStoreTestBase.java | 2 +- store/src/test/resources/logback-test.xml | 2 +- .../org/apache/rocketmq/test/factory/ConsumerFactory.java | 2 ++ .../org/apache/rocketmq/test/factory/ProducerFactory.java | 2 ++ .../main/java/org/apache/rocketmq/test/util/MQAdmin.java | 2 ++ .../test/java/org/apache/rocketmq/test/base/BaseConf.java | 2 +- .../org/apache/rocketmq/test/base/IntegrationTestBase.java | 6 +++--- .../test/base/dledger/DLedgerProduceAndConsumeIT.java | 7 +++++-- .../consumer/broadcast/order/OrderMsgBroadCastIT.java | 5 +++++ .../org/apache/rocketmq/test/delay/NormalMsgDelayIT.java | 3 ++- 14 files changed, 32 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a5f47e597c..dc53b3afc7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ ## Apache RocketMQ [![Build Status](https://travis-ci.org/apache/rocketmq.svg?branch=master)](https://travis-ci.org/apache/rocketmq) [![Coverage Status](https://coveralls.io/repos/github/apache/rocketmq/badge.svg?branch=master)](https://coveralls.io/github/apache/rocketmq?branch=master) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.rocketmq/rocketmq-all/badge.svg)](http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq) -[![GitHub release](https://img.shields.io/badge/release-download-orange.svg)](https://rocketmq.apache.org/dowloading/releases) +[![GitHub release](https://img.shields.io/badge/release-download-orange.svg)] +(https://rocketmq.apache.org/dowloading/releases) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) **[Apache RocketMQ](https://rocketmq.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.** @@ -47,3 +48,5 @@ We always welcome new contributions, whether for trivial cleanups, big new featu ---------- ## License [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) Apache Software Foundation + + diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java index b5e608558c..971237929a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ClientManageProcessor.java @@ -23,7 +23,6 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.filter.ExpressionType; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.body.CheckClientRequestBody; @@ -36,6 +35,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.filter.FilterFactory; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 9ffaed0a4f..4b055e0fc2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -548,10 +548,10 @@ public class MQClientInstance { } } catch (Exception e) { if (this.isBrokerInNameServer(addr)) { - log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr); + log.info("send heart beat to broker[{} {} {}] failed", brokerName, id, addr, e); } else { log.info("send heart beat to broker[{} {} {}] exception, because the broker not up, forget it", brokerName, - id, addr); + id, addr, e); } } } diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java index 66f7f5dc8f..78457dbfe8 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreShutDownTest.java @@ -42,9 +42,9 @@ public class DefaultMessageStoreShutDownTest { public void init() throws Exception { messageStore = spy(buildMessageStore()); boolean load = messageStore.load(); - when(messageStore.dispatchBehindBytes()).thenReturn(100L); assertTrue(load); messageStore.start(); + when(messageStore.dispatchBehindBytes()).thenReturn(100L); } @Test diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index fb9a7798df..78a6837727 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -115,7 +115,7 @@ public class MessageStoreTestBase extends StoreTestBase { protected void doGetMessages(MessageStore messageStore, String topic, int queueId, int num, long beginLogicsOffset) { for (int i = 0; i < num; i++) { - GetMessageResult getMessageResult = messageStore.getMessage("group", topic, 0, beginLogicsOffset + i, 3, null); + GetMessageResult getMessageResult = messageStore.getMessage("group", topic, queueId, beginLogicsOffset + i, 3, null); Assert.assertNotNull(getMessageResult); Assert.assertTrue(!getMessageResult.getMessageBufferList().isEmpty()); MessageExt messageExt = MessageDecoder.decode(getMessageResult.getMessageBufferList().get(0)); diff --git a/store/src/test/resources/logback-test.xml b/store/src/test/resources/logback-test.xml index 875b6715ac..a033816dda 100644 --- a/store/src/test/resources/logback-test.xml +++ b/store/src/test/resources/logback-test.xml @@ -28,7 +28,7 @@ - + diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java index c37cf1f96b..4850846266 100644 --- a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java +++ b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.factory; +import java.util.UUID; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer; @@ -64,6 +65,7 @@ public class ConsumerFactory { public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception { DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); + defaultMQPullConsumer.setInstanceName(UUID.randomUUID().toString()); defaultMQPullConsumer.setNamesrvAddr(nsAddr); defaultMQPullConsumer.start(); return defaultMQPullConsumer; diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java index 66767cc9f0..76e6e09d0b 100644 --- a/test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java +++ b/test/src/main/java/org/apache/rocketmq/test/factory/ProducerFactory.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.factory; +import java.util.UUID; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.test.util.RandomUtil; @@ -25,6 +26,7 @@ public class ProducerFactory { public static DefaultMQProducer getRMQProducer(String ns) { DefaultMQProducer producer = new DefaultMQProducer(RandomUtil.getStringByUUID()); + producer.setInstanceName(UUID.randomUUID().toString()); producer.setNamesrvAddr(ns); try { producer.start(); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java index bd151d0561..8863ee3e52 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.test.util; import java.util.HashMap; import java.util.Set; +import java.util.UUID; import org.apache.log4j.Logger; import org.apache.rocketmq.common.admin.TopicStatsTable; import org.apache.rocketmq.common.protocol.body.ClusterInfo; @@ -40,6 +41,7 @@ public class MQAdmin { int queueNum, int waitTimeSec) { boolean createResult = false; DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); + mqAdminExt.setInstanceName(UUID.randomUUID().toString()); mqAdminExt.setNamesrvAddr(nameSrvAddr); try { mqAdminExt.start(); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index 1f4fe7175e..d9955c04de 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -39,7 +39,7 @@ public class BaseConf { protected static String clusterName; protected static int brokerNum; protected static int waitTime = 5; - protected static int consumeTime = 5 * 60 * 1000; + protected static int consumeTime = 2 * 60 * 1000; protected static NamesrvController namesrvController; protected static BrokerController brokerController1; protected static BrokerController brokerController2; diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 0ac321e4a3..0eacd584fa 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -50,10 +50,10 @@ public class IntegrationTestBase { protected static final int COMMIT_LOG_SIZE = 1024 * 1024 * 100; protected static final int INDEX_NUM = 1000; - private static final AtomicInteger port = new AtomicInteger(50000); + private static final AtomicInteger port = new AtomicInteger(40000); public static synchronized int nextPort() { - return port.addAndGet(5); + return port.addAndGet(random.nextInt(10) + 10); } protected static Random random = new Random(); @@ -110,7 +110,7 @@ public class IntegrationTestBase { namesrvConfig.setKvConfigPath(baseDir + SEP + "namesrv" + SEP + "kvConfig.json"); namesrvConfig.setConfigStorePath(baseDir + SEP + "namesrv" + SEP + "namesrv.properties"); - nameServerNettyServerConfig.setListenPort(9000 + random.nextInt(1000)); + nameServerNettyServerConfig.setListenPort(nextPort()); NamesrvController namesrvController = new NamesrvController(namesrvConfig, nameServerNettyServerConfig); try { Assert.assertTrue(namesrvController.initialize()); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java index 813447f21f..7fd229749f 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java @@ -19,12 +19,13 @@ import org.apache.rocketmq.test.base.IntegrationTestBase; import org.apache.rocketmq.test.factory.ConsumerFactory; import org.apache.rocketmq.test.factory.ProducerFactory; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import static org.apache.rocketmq.test.base.IntegrationTestBase.nextPort; import static sun.util.locale.BaseLocale.SEP; -public class ProduceAndConsumeTest { +public class DLedgerProduceAndConsumeIT { public BrokerConfig buildBrokerConfig(String cluster, String brokerName) { BrokerConfig brokerConfig = new BrokerConfig(); @@ -58,7 +59,7 @@ public class ProduceAndConsumeTest { BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName); MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId); BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig); - Thread.sleep(1000); + Thread.sleep(3000); Assert.assertEquals(BrokerRole.SYNC_MASTER, storeConfig.getBrokerRole()); @@ -97,6 +98,8 @@ public class ProduceAndConsumeTest { Assert.assertArrayEquals(("Hello" + i).getBytes(), messageExt.getBody()); } + producer.shutdown(); + consumer.shutdown(); brokerController.shutdown(); } } diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java index 9294c3fd93..1d7ea2033a 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/broadcast/order/OrderMsgBroadCastIT.java @@ -29,10 +29,15 @@ import org.apache.rocketmq.test.util.TestUtils; import org.apache.rocketmq.test.util.VerifyUtils; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import static com.google.common.truth.Truth.assertThat; +/** + * Currently, dose not support the ordered broadcast message + */ +@Ignore public class OrderMsgBroadCastIT extends BaseBroadCastIT { private static Logger logger = Logger.getLogger(OrderMsgBroadCastIT.class); private RMQNormalProducer producer = null; diff --git a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java index 8cb0f41c1c..b97b0637b0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/delay/NormalMsgDelayIT.java @@ -51,7 +51,8 @@ public class NormalMsgDelayIT extends DelayConf { } @Test - public void testDelayLevell() { + public void testDelayLevel1() throws Exception { + Thread.sleep(3000); int delayLevel = 1; List delayMsgs = MQMessageFactory.getDelayMsg(topic, delayLevel, msgSize); producer.send(delayMsgs); From 1c2cbfa05624686a5482fddb313931f8351fe2ff Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 20 Dec 2018 15:56:58 +0800 Subject: [PATCH 047/122] Refresh the message store config to dledger config --- .../store/dledger/DLedgerCommitLog.java | 35 ++++++++++--------- .../store/dledger/MixCommitlogTest.java | 8 +++-- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 831f778310..b6aaa5b047 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -70,11 +70,6 @@ public class DLedgerCommitLog extends CommitLog { private long dividedCommitlogOffset = -1; - //The old commitlog should be deleted before the dledger commitlog - private final boolean originalDledgerEnableForceClean; - private final AtomicBoolean hasSetOriginalDledgerEnableForceClean = new AtomicBoolean(false); - - private boolean isInrecoveringOldCommitlog = false; public DLedgerCommitLog(final DefaultMessageStore defaultMessageStore) { @@ -87,7 +82,7 @@ public class DLedgerCommitLog extends CommitLog { dLedgerConfig.setStoreBaseDir(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir()); dLedgerConfig.setMappedFileSizeForEntryData(defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog()); dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen()); - originalDledgerEnableForceClean = dLedgerConfig.isEnableDiskForceClean(); + dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1); id = Integer.valueOf(dLedgerConfig.getSelfId().substring(1)) + 1; dLedgerServer = new DLedgerServer(dLedgerConfig); dLedgerFileStore = (DLedgerMmapFileStore) dLedgerServer.getdLedgerStore(); @@ -112,6 +107,17 @@ public class DLedgerCommitLog extends CommitLog { return true; } + private void refreshConfig() { + dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable()); + dLedgerConfig.setDeleteWhen(defaultMessageStore.getMessageStoreConfig().getDeleteWhen()); + dLedgerConfig.setFileReservedHours(defaultMessageStore.getMessageStoreConfig().getFileReservedTime() + 1); + } + + private void disableDeleteDledger() { + dLedgerConfig.setEnableDiskForceClean(false); + dLedgerConfig.setFileReservedHours(Integer.MAX_VALUE); + } + @Override public void start() { dLedgerServer.startup(); @@ -177,11 +183,11 @@ public class DLedgerCommitLog extends CommitLog { final boolean cleanImmediately ) { if (mappedFileQueue.getMappedFiles().isEmpty()) { - if (hasSetOriginalDledgerEnableForceClean.compareAndSet(false, true)) { - dLedgerConfig.setEnableDiskForceClean(originalDledgerEnableForceClean); - } + refreshConfig(); //To prevent too much log in defaultMessageStore return Integer.MAX_VALUE; + } else { + disableDeleteDledger(); } int count = super.deleteExpiredFile(expiredTime, deleteFilesInterval, intervalForcibly, cleanImmediately); if (count > 0 || mappedFileQueue.getMappedFiles().size() != 1) { @@ -251,16 +257,14 @@ public class DLedgerCommitLog extends CommitLog { return null; } - private void recover(long maxPhyOffsetOfConsumeQueue, boolean lastOk) { + private void recover(long maxPhyOffsetOfConsumeQueue) { dLedgerFileStore.load(); if (dLedgerFileList.getMappedFiles().size() > 0) { dLedgerFileStore.recover(); dividedCommitlogOffset = dLedgerFileList.getFirstMappedFile().getFileFromOffset(); MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile != null) { - dLedgerConfig.setEnableDiskForceClean(false); - } else { - hasSetOriginalDledgerEnableForceClean.set(true); + disableDeleteDledger(); } long maxPhyOffset = dLedgerFileList.getMaxWrotePosition(); // Clear ConsumeQueue redundant data @@ -277,7 +281,6 @@ public class DLedgerCommitLog extends CommitLog { isInrecoveringOldCommitlog = false; MappedFile mappedFile = this.mappedFileQueue.getLastMappedFile(); if (mappedFile == null) { - hasSetOriginalDledgerEnableForceClean.set(true); return; } ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); @@ -309,12 +312,12 @@ public class DLedgerCommitLog extends CommitLog { @Override public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { - recover(maxPhyOffsetOfConsumeQueue, true); + recover(maxPhyOffsetOfConsumeQueue); } @Override public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { - recover(maxPhyOffsetOfConsumeQueue, false); + recover(maxPhyOffsetOfConsumeQueue); } @Override diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index 8fb62f3b23..e09f579e2a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -133,6 +133,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { { DefaultMessageStore dledgerStore = createDledgerMessageStore(base, group, "n0", peers, null, true, 0); DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); + Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable()); Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); Thread.sleep(2000); @@ -143,6 +144,7 @@ public class MixCommitlogTest extends MessageStoreTestBase { Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); Assert.assertEquals(0, dledgerStore.getMinPhyOffset()); maxPhysicalOffset = dledgerStore.getMaxPhyOffset(); + Assert.assertTrue(maxPhysicalOffset > 0); doGetMessages(dledgerStore, topic, 0, 2000, 0); @@ -156,13 +158,15 @@ public class MixCommitlogTest extends MessageStoreTestBase { } Assert.assertEquals(dividedOffset, dledgerStore.getMinPhyOffset()); Assert.assertEquals(maxPhysicalOffset, dledgerStore.getMaxPhyOffset()); + + Assert.assertTrue(dledgerStore.getMessageStoreConfig().isCleanFileForciblyEnable()); Assert.assertTrue(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); - dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().setEnableDiskForceClean(false); + //Test fresh + dledgerStore.getMessageStoreConfig().setCleanFileForciblyEnable(false); for (int i = 0; i < 100; i++) { Assert.assertEquals(Integer.MAX_VALUE, dledgerStore.getCommitLog().deleteExpiredFile(System.currentTimeMillis(), 0, 0, true)); } - //should not change the value Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); doGetMessages(dledgerStore, topic, 0, 1000, 1000); dledgerStore.shutdown(); From 55eda18472a67b267b689d549ae209f883c0ce6a Mon Sep 17 00:00:00 2001 From: dongeforever Date: Fri, 21 Dec 2018 10:57:06 +0800 Subject: [PATCH 048/122] Fix style --- .../java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 1 - 1 file changed, 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index b6aaa5b047..3e1d25414a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -31,7 +31,6 @@ import io.openmessaging.storage.dledger.utils.DLedgerUtils; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; From e1494951e933372e6d524feeb21f38b892d2ed55 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 24 Dec 2018 16:51:39 +0800 Subject: [PATCH 049/122] Fix the initial reput offset, according to the maxPhisicalPosInLogicQueue --- .../rocketmq/store/DefaultMessageStore.java | 43 ++++++++++++------- .../store/dledger/DLedgerCommitLog.java | 10 ++--- .../store/dledger/MixCommitlogTest.java | 2 + 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 124e14e780..254f87a652 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -224,8 +224,10 @@ public class DefaultMessageStore implements MessageStore { lockFile.getChannel().force(true); { /** - * 1. calculate the reput offset according to the consume queue; - * 2. make sure the lagged messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed. + * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog; + * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go; + * 3. Calculate the reput offset according to the consume queue; + * 4. Make sure the fall-behind messages to be dispatched before starting the commitlog, especially when the broker role are automatically changed. */ long maxPhysicalPosInLogicQueue = commitLog.getMinOffset(); for (ConcurrentMap maps : this.consumeQueueTable.values()) { @@ -235,23 +237,34 @@ public class DefaultMessageStore implements MessageStore { } } } - log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMaxOffset={} clConfirmedOffset={}", - maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); - long reputOffset; - if (this.getMessageStoreConfig().isDuplicationEnable()) { - reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getConfirmOffset()); - } else { - reputOffset = Math.min(maxPhysicalPosInLogicQueue, this.commitLog.getMaxOffset()); + if (maxPhysicalPosInLogicQueue < 0) { + maxPhysicalPosInLogicQueue = 0; } - if (reputOffset < 0) { - reputOffset = 0; + if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) { + maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset(); + /** + * This happens in following conditions: + * 1. If someone removes all the consumequeue files or the disk get damaged. + * 2. Launch a new broker, and copy the commitlog from other brokers. + * + * All the conditions has the same in common that the maxPhysicalPosInLogicQueue should be 0. + * If the maxPhysicalPosInLogicQueue is gt 0, there maybe something wrong. + */ + log.warn("[TooSmallCqOffset] maxPhysicalPosInLogicQueue={} clMinOffset={}", maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset()); } - this.reputMessageService.setReputFromOffset(reputOffset); + log.info("[SetReputOffset] maxPhysicalPosInLogicQueue={} clMinOffset={} clMaxOffset={} clConfirmedOffset={}", + maxPhysicalPosInLogicQueue, this.commitLog.getMinOffset(), this.commitLog.getMaxOffset(), this.commitLog.getConfirmOffset()); + this.reputMessageService.setReputFromOffset(maxPhysicalPosInLogicQueue); this.reputMessageService.start(); - //Finish dispatching the messages fall behind - //Note that, if dledger is enabled, the maxOffset maybe -1, so here only require the dispatchBehindBytes > 0 - while (this.dispatchBehindBytes() > 0) { + /** + * 1. Finish dispatching the messages fall behind, then to start other services. + * 2. DLedger committedPos may be missing, so here just require dispatchBehindBytes <= 0 + */ + while (true) { + if (dispatchBehindBytes() <= 0) { + break; + } Thread.sleep(1000); log.info("Try to finish doing reput the messages fall behind during the starting, reputOffset={} maxOffset={} behind={}", this.reputMessageService.getReputFromOffset(), this.getMaxPhyOffset(), this.dispatchBehindBytes()); } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 3e1d25414a..1f029c51a0 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -135,13 +135,13 @@ public class DLedgerCommitLog extends CommitLog { @Override public long getMaxOffset() { - if (dividedCommitlogOffset > 0 && dLedgerFileStore.getCommittedPos() < 0) { - return dividedCommitlogOffset; + if (dLedgerFileStore.getCommittedPos() > 0) { + return dLedgerFileStore.getCommittedPos(); } - if (dLedgerFileStore.getCommittedPos() == -1) { - return 0; + if (dLedgerFileList.getMinOffset() > 0) { + return dLedgerFileList.getMinOffset(); } - return dLedgerFileStore.getCommittedPos(); + return 0; } @Override diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index e09f579e2a..540486d2aa 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -85,6 +85,8 @@ public class MixCommitlogTest extends MessageStoreTestBase { DLedgerCommitLog dLedgerCommitLog = (DLedgerCommitLog) dledgerStore.getCommitLog(); Assert.assertFalse(dLedgerCommitLog.getdLedgerServer().getdLedgerConfig().isEnableDiskForceClean()); Assert.assertEquals(dividedOffset, dLedgerCommitLog.getDividedCommitlogOffset()); + Assert.assertEquals(0, dledgerStore.dispatchBehindBytes()); + Assert.assertEquals(dividedOffset, dLedgerCommitLog.getMaxOffset()); Thread.sleep(2000); doPutMessages(dledgerStore, topic, 0, 1000, 1000); Thread.sleep(500); From 3f0b0b231eb3cce7e9f33e8bade21c44740f5301 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Mon, 24 Dec 2018 17:38:31 +0800 Subject: [PATCH 050/122] Revise the reputoffset if it is smaller that minOffset of the commitlog --- .../java/org/apache/rocketmq/store/DefaultMessageStore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 254f87a652..6cec58d861 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -1810,6 +1810,11 @@ public class DefaultMessageStore implements MessageStore { } private void doReput() { + if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) { + log.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.", + this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); + this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); + } for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { if (DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() From 28a0906bcfdc37b150a90b20655bd997e10cd25f Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 26 Dec 2018 14:25:55 +0800 Subject: [PATCH 051/122] Polish file reserved hours to 24 * 365 * 10 when disableDeleteDLedger --- .../org/apache/rocketmq/store/dledger/DLedgerCommitLog.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 1f029c51a0..a48c1bed8b 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -75,6 +75,7 @@ public class DLedgerCommitLog extends CommitLog { super(defaultMessageStore); dLedgerConfig = new DLedgerConfig(); dLedgerConfig.setEnableDiskForceClean(defaultMessageStore.getMessageStoreConfig().isCleanFileForciblyEnable()); + dLedgerConfig.setStoreType(DLedgerConfig.FILE); dLedgerConfig.setSelfId(defaultMessageStore.getMessageStoreConfig().getdLegerSelfId()); dLedgerConfig.setGroup(defaultMessageStore.getMessageStoreConfig().getdLegerGroup()); dLedgerConfig.setPeers(defaultMessageStore.getMessageStoreConfig().getdLegerPeers()); @@ -114,7 +115,7 @@ public class DLedgerCommitLog extends CommitLog { private void disableDeleteDledger() { dLedgerConfig.setEnableDiskForceClean(false); - dLedgerConfig.setFileReservedHours(Integer.MAX_VALUE); + dLedgerConfig.setFileReservedHours(24 * 365 * 10); } @Override From 7e8c01255e67c69b49e0d18e5198d668ebe3b3b9 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Wed, 26 Dec 2018 14:35:14 +0800 Subject: [PATCH 052/122] Fix typo --- .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 3418e82094..7a5a96e9ea 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -237,7 +237,7 @@ public class BrokerController { this.brokerConfig); if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); - ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLegerLeaderElector().addRoleChangeHandler(roleChangeHandler); + ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin From 89c0668c0fbe599a0d1c65dced8fe1648bbaffff Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 27 Dec 2018 16:56:57 +0800 Subject: [PATCH 053/122] Add example conf and bin for dledger --- distribution/bin/dledger/fast-try.sh | 87 ++++++++++++++++++++++++ distribution/conf/dledger/broker-n0.conf | 27 ++++++++ distribution/conf/dledger/broker-n1.conf | 27 ++++++++ distribution/conf/dledger/broker-n2.conf | 27 ++++++++ distribution/release.xml | 2 +- 5 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 distribution/bin/dledger/fast-try.sh create mode 100644 distribution/conf/dledger/broker-n0.conf create mode 100644 distribution/conf/dledger/broker-n1.conf create mode 100644 distribution/conf/dledger/broker-n2.conf diff --git a/distribution/bin/dledger/fast-try.sh b/distribution/bin/dledger/fast-try.sh new file mode 100644 index 0000000000..e536d455fa --- /dev/null +++ b/distribution/bin/dledger/fast-try.sh @@ -0,0 +1,87 @@ +#! /bin/bash + +## Revise the base dir +CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)" +RMQ_DIR=$CURRENT_DIR/../.. +cd $RMQ_DIR + +function startNameserver() { + export JAVA_OPT_EXT=" -Xms512m -Xmx512m " + nohup bin/mqnamesrv & +} + +function startBroker() { + export JAVA_OPT_EXT=" -Xms1g -Xmx1g " + conf_name=$1 + nohup bin/mqbroker -c $conf_name & +} + +function stopNameserver() { + PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}') + if [ ! -z "$PIDS" ]; then + kill -s TERM $PIDS + fi +} + +function stopBroker() { + conf_name=$1 + PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') + i=1 + while [ ! -z "$PIDS" -a $i -lt 5 ] + do + echo "Waiting to kill ..." + kill -s TERM $PIDS + ((i=$i+1)) + sleep 2 + PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') + done + PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') + if [ ! -z "$PIDS" ]; then + kill -9 $PIDS + fi +} + +function stopAll() { + ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill + stopNameserver + stopBroker ./conf/dledger/broker-n0.conf + stopBroker ./conf/dledger/broker-n1.conf + stopBroker ./conf/dledger/broker-n2.conf +} + +function startAll() { + startNameserver + startBroker ./conf/dledger/broker-n0.conf + startBroker ./conf/dledger/broker-n1.conf + startBroker ./conf/dledger/broker-n2.conf +} + +function checkConf() { + if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then + echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists" + exit -1 + fi +} + + + +## Main +if [ $# -lt 1 ]; then + echo "Usage: sh $0 start|stop" + exit -1 +fi +action=$1 +checkConf +case $action in + "start") + startAll + exit + ;; + "stop") + stopAll + ;; + *) + echo "Usage: sh $0 start|stop" + ;; +esac + diff --git a/distribution/conf/dledger/broker-n0.conf b/distribution/conf/dledger/broker-n0.conf new file mode 100644 index 0000000000..5351e497dc --- /dev/null +++ b/distribution/conf/dledger/broker-n0.conf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName = RaftCluster +brokerName=RaftNode00 +listenPort=30911 +namesrvAddr=127.0.0.1:9876 +storePathRootDir=/tmp/rmqstore/node00 +storePathCommitLog=/tmp/rmqstore/node00/commitlog +enableDLegerCommitLog=true +dLegerGroup=RaftNode00 +dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 +## must be unique +dLegerSelfId=n0 +sendMessageThreadPoolNums=16 diff --git a/distribution/conf/dledger/broker-n1.conf b/distribution/conf/dledger/broker-n1.conf new file mode 100644 index 0000000000..6aaf8f9309 --- /dev/null +++ b/distribution/conf/dledger/broker-n1.conf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName = RaftCluster +brokerName=RaftNode00 +listenPort=30921 +namesrvAddr=127.0.0.1:9876 +storePathRootDir=/tmp/rmqstore/node01 +storePathCommitLog=/tmp/rmqstore/node01/commitlog +enableDLegerCommitLog=true +dLegerGroup=RaftNode00 +dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 +## must be unique +dLegerSelfId=n1 +sendMessageThreadPoolNums=16 diff --git a/distribution/conf/dledger/broker-n2.conf b/distribution/conf/dledger/broker-n2.conf new file mode 100644 index 0000000000..c863d89ee8 --- /dev/null +++ b/distribution/conf/dledger/broker-n2.conf @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName = RaftCluster +brokerName=RaftNode00 +listenPort=30931 +namesrvAddr=127.0.0.1:9876 +storePathRootDir=/tmp/rmqstore/node02 +storePathCommitLog=/tmp/rmqstore/node02/commitlog +enableDLegerCommitLog=true +dLegerGroup=RaftNode00 +dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 +## must be unique +dLegerSelfId=n2 +sendMessageThreadPoolNums=16 diff --git a/distribution/release.xml b/distribution/release.xml index 76bf5fbb87..ae9eb08187 100644 --- a/distribution/release.xml +++ b/distribution/release.xml @@ -40,7 +40,7 @@ - bin/* + bin/** 0755 From 0d395694f9644499fc803151e1f652fc84714624 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 27 Dec 2018 19:48:31 +0800 Subject: [PATCH 054/122] Add docs --- docs/cn/dledger/quick_start.md | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 docs/cn/dledger/quick_start.md diff --git a/docs/cn/dledger/quick_start.md b/docs/cn/dledger/quick_start.md new file mode 100644 index 0000000000..eddb7833fd --- /dev/null +++ b/docs/cn/dledger/quick_start.md @@ -0,0 +1,54 @@ +### 前言 +该文档主要介绍如何快速构建和部署基于 DLedger 的可以自动容灾切换的 RocketMQ 集群。 + + +### 源码构建 +构建分为两个部分，需要先构建 DLedger，然后 构建 RocketMQ + +#### 构建 DLedger + +`git clone https://github.com/openmessaging/openmessaging-storage-dledger.git` + +`cd openmessaging-storage-dledger` + +`mvn clean install -DskipTests` + +#### 构建 RocketMQ + +`git clone https://github.com/apache/rocketmq.git` + +`cd rocketmq` + +`git checkout -b store_with_dledger origin/store_with_dledger` + +`mvn -Prelease-all -DskipTests clean install -U` + +### 快速部署 + +在构建成功后 + +`cd distribution/target/apache-rocketmq` + +`sh bin/dledger/fast-try.sh start` + +如果上面的步骤执行成功，可以通过 mqadmin 运维命令查看集群状态。 + +`sh bin/mqadmin clusterList -n 127.0.0.1:9876` + +顺利的话，会看到如下内容： + +![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa) + +启动成功，现在可以向集群收发消息进行验证了。 + +关闭快速集群，可以执行： + +`sh bin/dledger/fast-try.sh stop` + +快速部署，默认配置在 conf/dledger 里面，默认的存储路径在 /tmp/rmqstore。 + + +### + + + From d097acbbdd60ee6145d7a0ae7224cca5ef3471ac Mon Sep 17 00:00:00 2001 From: dongeforever Date: Sat, 29 Dec 2018 18:06:37 +0800 Subject: [PATCH 055/122] Add cn deploy guide for rocketmq cluster --- docs/cn/dledger/deploy_guide.md | 76 +++++++++++++++++++++++++++++++++ docs/cn/dledger/quick_start.md | 19 ++++++--- 2 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 docs/cn/dledger/deploy_guide.md diff --git a/docs/cn/dledger/deploy_guide.md b/docs/cn/dledger/deploy_guide.md new file mode 100644 index 0000000000..cabfcb0e7c --- /dev/null +++ b/docs/cn/dledger/deploy_guide.md @@ -0,0 +1,76 @@ +## 前言 +该文档主要介绍如何部署自动容灾切换的 RocketMQ-on-DLedger Group。 + +RocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。 +RocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。 +RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。 + +## 1. 新集群部署 + +#### 1.1 编写配置 +每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。 +编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。 +关键配置介绍： + +| name | 含义 | 举例 | +| --- | --- | --- | +| enableDLegerCommitLog | 是否启动 DLedger  | true | +| dLegerGroup | DLedger Raft Group的名字，建议和 brokerName 保持一致 | RaftNode00 | +| dLegerPeers | DLedger Group 内各节点的端口信息，同一个 Group 内的各个节点配置必须要保证一致 | n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 | +| dLegerSelfId | 节点 id, 必须属于 dLegerPeers 中的一个；同 Group 内各个节点要唯一 | n0 | +| sendMessageThreadPoolNums | 发送线程个数，建议配置成 Cpu 核数 | 16 | + +这里贴出 conf/dledger/broker-n0.conf 的配置举例。 + +``` +brokerClusterName = RaftCluster +brokerName=RaftNode00 +listenPort=30911 +namesrvAddr=127.0.0.1:9876 +storePathRootDir=/tmp/rmqstore/node00 +storePathCommitLog=/tmp/rmqstore/node00/commitlog +enableDLegerCommitLog=true +dLegerGroup=RaftNode00 +dLegerPeers=n0-127.0.0.1:40911;n1-127.0.0.1:40912;n2-127.0.0.1:40913 +## must be unique +dLegerSelfId=n0 +sendMessageThreadPoolNums=16 +``` + +### 1.2 启动 Broker + +与老版本的启动方式一致。 + +`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & ` +`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & ` +`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & ` + + +## 2. 旧集群升级 + +如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。 +如果旧集群采用 Master-Slave 方式部署，则每个 Master-Slave 组都需要转换成一个 RocketMQ-on-DLedger Group。 + +### 2.1 杀掉旧的 Broker + +可以通过 kill 命令来完成，也可以调用 `bin/mqshutdown broker`。 + +### 2.2 检查旧的 Commitlog + +RocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。 +如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过 md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。 + +虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。 +所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。 + +在 3 台机器准备好了之后，旧 Commitlog 文件也保证一致之后，就可以开始走下一步修改配置了。 + +### 2.3 修改配置 + +参考新集群部署。 + +### 2.4 重新启动 Broker + +参考新集群部署。 + + diff --git a/docs/cn/dledger/quick_start.md b/docs/cn/dledger/quick_start.md index eddb7833fd..3d1989a52c 100644 --- a/docs/cn/dledger/quick_start.md +++ b/docs/cn/dledger/quick_start.md @@ -1,11 +1,12 @@ ### 前言 该文档主要介绍如何快速构建和部署基于 DLedger 的可以自动容灾切换的 RocketMQ 集群。 +详细的新集群部署和旧集群升级指南请参考 [部署指南](deploy_guide.md)。 -### 源码构建 +### 1. 源码构建 构建分为两个部分，需要先构建 DLedger，然后 构建 RocketMQ -#### 构建 DLedger +#### 1.1 构建 DLedger `git clone https://github.com/openmessaging/openmessaging-storage-dledger.git` @@ -13,7 +14,7 @@ `mvn clean install -DskipTests` -#### 构建 RocketMQ +#### 1.2 构建 RocketMQ `git clone https://github.com/apache/rocketmq.git` @@ -23,7 +24,7 @@ `mvn -Prelease-all -DskipTests clean install -U` -### 快速部署 +### 2. 快速部署 在构建成功后 @@ -39,7 +40,9 @@ ![ClusterList](https://img.alicdn.com/5476e8b07b923/TB11Z.ZyCzqK1RjSZFLXXcn2XXa) -启动成功，现在可以向集群收发消息进行验证了。 +（BID 为 0 的表示 Master，其余都是 Follower） + +启动成功，现在可以向集群收发消息，并进行容灾切换测试了。 关闭快速集群，可以执行： @@ -48,7 +51,11 @@ 快速部署，默认配置在 conf/dledger 里面，默认的存储路径在 /tmp/rmqstore。 -### +### 3. 容灾切换 + +部署成功，杀掉 Leader 之后（在上面的例子中，杀掉端口 30931 所在的进程），等待约 10s 左右，用 clusterList 命令查看集群，就会发现 Leader 切换到另一个节点了。 + + From 8cec19411422e4e419c1236c3a3faba5b9e99b6c Mon Sep 17 00:00:00 2001 From: dongeforever Date: Sat, 29 Dec 2018 18:09:15 +0800 Subject: [PATCH 056/122] Polish new line for docs --- docs/cn/dledger/deploy_guide.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/cn/dledger/deploy_guide.md b/docs/cn/dledger/deploy_guide.md index cabfcb0e7c..faebb96e59 100644 --- a/docs/cn/dledger/deploy_guide.md +++ b/docs/cn/dledger/deploy_guide.md @@ -1,16 +1,16 @@ ## 前言 该文档主要介绍如何部署自动容灾切换的 RocketMQ-on-DLedger Group。 -RocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。 -RocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。 -RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。 +RocketMQ-on-DLedger Group 是指一组**相同名称**的 Broker，至少需要 3 个节点，通过 Raft 自动选举出一个 Leader，其余节点 作为 Follower，并在 Leader 和 Follower 之间复制数据以保证高可用。 +RocketMQ-on-DLedger Group 能自动容灾切换，并保证数据一致。 +RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多个 RocketMQ-on-DLedger Group 同时对外提供服务。 ## 1. 新集群部署 #### 1.1 编写配置 -每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。 -编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。 -关键配置介绍： +每个 RocketMQ-on-DLedger Group 至少准备三台机器（本文假设为 3）。 +编写 3 个配置文件，建议参考 conf/dledger 目录下的配置文件样例。 +关键配置介绍： | name | 含义 | 举例 | | --- | --- | --- | @@ -20,7 +20,7 @@ RocketMQ-on-DLedger Group 是可以水平扩展的，也即可以部署任意多 | dLegerSelfId | 节点 id, 必须属于 dLegerPeers 中的一个；同 Group 内各个节点要唯一 | n0 | | sendMessageThreadPoolNums | 发送线程个数，建议配置成 Cpu 核数 | 16 | -这里贴出 conf/dledger/broker-n0.conf 的配置举例。 +这里贴出 conf/dledger/broker-n0.conf 的配置举例。 ``` brokerClusterName = RaftCluster @@ -41,14 +41,14 @@ sendMessageThreadPoolNums=16 与老版本的启动方式一致。 -`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & ` -`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & ` -`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & ` +`nohup sh bin/mqbroker -c conf/dledger/xxx-n0.conf & ` +`nohup sh bin/mqbroker -c conf/dledger/xxx-n1.conf & ` +`nohup sh bin/mqbroker -c conf/dledger/xxx-n2.conf & ` ## 2. 旧集群升级 -如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。 +如果旧集群采用 Master 方式部署，则每个 Master 都需要转换成一个 RocketMQ-on-DLedger Group。 如果旧集群采用 Master-Slave 方式部署，则每个 Master-Slave 组都需要转换成一个 RocketMQ-on-DLedger Group。 ### 2.1 杀掉旧的 Broker @@ -57,11 +57,11 @@ sendMessageThreadPoolNums=16 ### 2.2 检查旧的 Commitlog -RocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。 -如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过 md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。 +RocketMQ-on-DLedger 组中的每个节点，可以兼容旧的 Commitlog ，但其 Raft 复制过程，只能针对新增加的消息。因此，为了避免出现异常，需要保证 旧的 Commitlog 是一致的。 +如果旧的集群是采用 Master-Slave 方式部署，有可能在shutdown时，其数据并不是一致的，建议通过md5sum 的方式，检查最近的最少 2 个 Commmitlog 文件，如果发现不一致，则通过拷贝的方式进行对齐。 -虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。 -所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。 +虽然 RocketMQ-on-DLedger Group 也可以以 2 节点方式部署，但其会丧失容灾切换能力（2n + 1 原则，至少需要3个节点才能容忍其中 1 个宕机）。 +所以在对齐了 Master 和 Slave 的 Commitlog 之后，还需要准备第 3 台机器，并把旧的 Commitlog 从 Master 拷贝到 第 3 台机器（记得同时拷贝一下 config 文件夹）。 在 3 台机器准备好了之后，旧 Commitlog 文件也保证一致之后，就可以开始走下一步修改配置了。 From 7574fde331424c08ef3e2da2960ebc9f74dc9c52 Mon Sep 17 00:00:00 2001 From: dinglei Date: Sat, 12 Jan 2019 22:52:07 +0800 Subject: [PATCH 057/122] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index dc53b3afc7..8f670c2f94 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ ## Apache RocketMQ [![Build Status](https://travis-ci.org/apache/rocketmq.svg?branch=master)](https://travis-ci.org/apache/rocketmq) [![Coverage Status](https://coveralls.io/repos/github/apache/rocketmq/badge.svg?branch=master)](https://coveralls.io/github/apache/rocketmq?branch=master) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.apache.rocketmq/rocketmq-all/badge.svg)](http://search.maven.org/#search%7Cga%7C1%7Corg.apache.rocketmq) -[![GitHub release](https://img.shields.io/badge/release-download-orange.svg)] -(https://rocketmq.apache.org/dowloading/releases) +[![GitHub release](https://img.shields.io/badge/release-download-orange.svg)](https://rocketmq.apache.org/dowloading/releases) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) **[Apache RocketMQ](https://rocketmq.apache.org) is a distributed messaging and streaming platform with low latency, high performance and reliability, trillion-level capacity and flexible scalability.** From cd4779e03a5ad1ed0360c729d1f8b80b015cf170 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 31 Jan 2019 15:51:47 +0800 Subject: [PATCH 058/122] Do not send heartbeat for every pull request --- .../client/impl/factory/MQClientInstance.java | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 88f06ea4ca..57d2eda186 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -1046,20 +1046,8 @@ public class MQClientInstance { if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) { return this.brokerVersionTable.get(brokerName).get(brokerAddr); } - } else { - HeartbeatData heartbeatData = prepareHeartbeatData(); - try { - int version = this.mQClientAPIImpl.sendHearbeat(brokerAddr, heartbeatData, 3000); - return version; - } catch (Exception e) { - if (this.isBrokerInNameServer(brokerAddr)) { - log.info("send heart beat to broker[{} {}] failed", brokerName, brokerAddr); - } else { - log.info("send heart beat to broker[{} {}] exception, because the broker not up, forget it", brokerName, - brokerAddr); - } - } } + //To do need to fresh the version return 0; } From f3130c3a3a86bb84f3d0b22d8c7b03a8a1984ef1 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Thu, 21 Feb 2019 16:40:55 +0800 Subject: [PATCH 059/122] Polish pom to use dledger 0.1 from central repo --- store/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/pom.xml b/store/pom.xml index 13e3b7d590..a1aaf39a41 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -31,7 +31,7 @@ io.openmessaging.storage dledger - 0.1-SNAPSHOT + 0.1 org.apache.rocketmq From 3b01e89a07feac330e5d0ee80917a2b722876327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E6=B5=AA?= Date: Thu, 28 Feb 2019 21:34:36 +0800 Subject: [PATCH 060/122] delete useless image references and correct the description --- docs/cn/concept.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/cn/concept.md b/docs/cn/concept.md index 38036f22f6..ef59050748 100644 --- a/docs/cn/concept.md +++ b/docs/cn/concept.md @@ -1,9 +1,7 @@ # 基本概念 ## 消息模型（Message Model） - - -RocketMQ 消息模型如图1所示，主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。 +RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer 负责生产消息，Consumer 负责消费消息，Broker 负责存储消息。Broker 在实际部署过程中对应一台服务器，每个 Broker 可以存储多个Topic的消息，每个Topic的消息也可以分片存储于不同的 Broker。Message Queue 用于存储消息的物理地址，每个Topic中的消息地址存储于多个 Message Queue 中。ConsumerGroup 由多个Consumer 实例构成。 ## 消息生产者（Producer） 负责生产消息，一般由业务系统负责生产消息。一个消息生产者会把业务应用系统里产生的消息发送到broker服务器。RocketMQ提供多种发送方式，同步发送、异步发送、顺序发送、单向发送。同步和异步方式均需要Broker返回确认信息，单向发送不需要。 ## 消息消费者（Consumer） From 579254d9a6c40b473b783a80da6d25b4a61f9bf0 Mon Sep 17 00:00:00 2001 From: Ly641921791 <641921791@qq.com> Date: Fri, 1 Mar 2019 18:53:10 +0800 Subject: [PATCH 061/122] [RIP-9] Modify the numbering style of the first paragraph --- docs/en/CLITools.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/CLITools.md b/docs/en/CLITools.md index 06b414b095..7a0173efcc 100644 --- a/docs/en/CLITools.md +++ b/docs/en/CLITools.md @@ -2,12 +2,12 @@ Before introducing the mqadmin management tool, the following points need to be declared: -1. The way of executing a command is:./mqadmin {command} {args} -2. Almost all commands need to attach the -n option to represent the nameServer address, formatted as ip:port; -3. Almost all commands can get help information with the -h option; -4. If the broker address -b option and clusterName -c option are both configured with specific values, the command execution will select the broker address specified by -b option. The value of the -b option can only be configured with a single address. The format is ip:port. The default port value is 10911. If the value of the -b option is not configured, the command will be applied to all brokers in the entire cluster. -5. You can see many commands under tools, but not all commands can be used, only the commands initialized in MQAdminStartup can be used, you can also modify this class, add or customize commands; -6. Due to the issue of version update, a small number of commands may not be updated in time, please read the related command source code to eliminate and resolve the error. +- The way of executing a command is:./mqadmin {command} {args} +- Almost all commands need to attach the -n option to represent the nameServer address, formatted as ip:port; +- Almost all commands can get help information with the -h option; +- If the broker address -b option and clusterName -c option are both configured with specific values, the command execution will select the broker address specified by -b option. The value of the -b option can only be configured with a single address. The format is ip:port. The default port value is 10911. If the value of the -b option is not configured, the command will be applied to all brokers in the entire cluster. +- You can see many commands under tools, but not all commands can be used, only the commands initialized in MQAdminStartup can be used, you can also modify this class, add or customize commands; +- Due to the issue of version update, a small number of commands may not be updated in time, please read the related command source code to eliminate and resolve the error. ## 1 Topic related command instructions From cb89eafa3c419ffc857217e13251b3ce5993b8b7 Mon Sep 17 00:00:00 2001 From: robertohuang <756858620@qq.com> Date: Sat, 2 Mar 2019 11:03:21 +0800 Subject: [PATCH 062/122] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=9B=A0=E4=B8=BA?= =?UTF-8?q?=E6=96=87=E6=A1=A3:Future.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/en/Feature.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/en/Feature.md b/docs/en/Feature.md index eb6a328f49..ed767f1dca 100644 --- a/docs/en/Feature.md +++ b/docs/en/Feature.md @@ -1,8 +1,8 @@ # Features -## Subscribe and Publish +## 1 Subscribe and Publish Message publication refers to that a producer sends messages to a topic; Message subscription means a consumer follows a topic with certain tags and then consumes data from that topic. -## Message Ordering +## 2 Message Ordering Message ordering refers to that a group of messages can be consumed orderly as they are published. For example, an order generates three messages: order creation, order payment, and order completion. It only makes sense to consume them in their generated order, but orders can be consumed in parallel at the same time. RocketMQ can strictly guarantee these messages are in order. Orderly message are divided into global orderly message and partitioned orderly message. Global order means that all messages under a certain topic must be in order, partitioned order only requires each group of messages are consumed orderly. @@ -13,10 +13,10 @@ Applicable scenario: the performance requirement is not high, and all messages a For a given Topic, all messages are partitioned according to sharding key. Messages within the same partition are published and consumed in strict FIFO order. Sharding key is the key field to distinguish message's partition, which is a completely different concept from the key of ordinary messages. Applicable scenario: high performance requirement, with sharding key as the partition field, messages within the same partition are published and consumed according to FIFO principle strictly. -## Message Filter +## 3 Message Filter Consumers of RocketMQ can filter messages based on tags as well as support for user-defined attribute filtering. Message filter is currently implemented on the Broker side, with the advantage of reducing the network transmission of useless messages for Consumer and the disadvantage of increasing the burden on the Broker and relatively complex implementation. -## Message Reliability +## 4 Message Reliability RocketMQ supports high reliability of messages in several situations: 1) Broker shutdown normally 2) Broker abnormal crash @@ -29,16 +29,16 @@ In the four cases of 1), 2), 3), and 4) where the hardware resource can be recov 5) and 6) are single point of failure and cannot be recovered. Once it happens, all messages on the single point will be lost. In both cases, RocketMQ ensures that 99% of the messages are not lost through asynchronous replication, but a very few number of messages may still be lost. Synchronous double write mode can completely avoid single point of failure, which will surely affect the performance and suitable for the occasion of high demand for message reliability, such as money related applications. Note: RocketMQ supports synchronous double writes since version 3.0. -## At Least Once +## 5 At Least Once At least Once refers to that every message will be delivered at least once. RocketMQ supports this feature because the Consumer pulls the message locally and does not send an ack back to the server until it has consumed it. -## Backtracking Consumption +## 6 Backtracking Consumption Backtracking consumption refers to that the Consumer has consumed the message successfully, but the business needs to consume again. To support this function, the message still needs to be retained after the Broker sends the message to the Consumer successfully. The re-consumption is normally based on time dimension. For example, after the recovery of the Consumer system failure, the data one hour ago needs to be re-consumed, then the Broker needs to provide a mechanism to reverse the consumption progress according to the time dimension. RocketMQ supports backtracking consumption by time trace, with the time dimension down to milliseconds. -## Transactional Message +## 7 Transactional Message RocketMQ transactional message refers to the fact that the application of a local transaction and the sending of a Message operation can be defined in a global transaction which means both succeed or fail simultaneously. RocketMQ transactional message provides distributed transaction functionality similar to X/Open XA, enabling the ultimate consistency of distributed transactions through transactional message. -## Scheduled Message +## 8 Scheduled Message Scheduled message(delay queue) refers to that messages are not consumed immediately after they are sent to the broker, but waiting to be delivered to the real topic after a specific time. The broker has a configuration item, messageDelayLevel, with default values “1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h”, 18 levels. Users can configure a custom messageDelayLevel. Note that messageDelayLevel is a broker's property rather than a topic's. When sending a message, just set the delayLevel level: msg.setDelayLevel(level). There are three types of levels: @@ -50,21 +50,21 @@ Scheduled messages are temporarily saved in a topic named SCHEDULE_TOPIC_XXXX, a Note that Scheduled messages are counted both the first time they are written and the time they are scheduled to be written to the real topic, so both the send number and the TPS are increased. -## Message Retry +## 9 Message Retry When the Consumer fails to consume the message, a retry mechanism is needed to make the message to be consumed again. Consumer's consume failure can usually be classified as follows: - due to the reasons of the message itself, such as deserialization failure, the message data itself cannot be processed (for example, the phone number of the current message is cancelled and cannot be charged), etc. This kind of error usually requires skipping this message and consuming others since immediately retry would be failed 99%, so it is better to provide a timed retry mechanism that retries after 10 seconds. - due to the reasons of dependent downstream application services are not available, such as db connection is not usable, perimeter network is not unreachable, etc. When this kind of error is encountered, consuming other messages will also result in an error even if the current failed message is skipped. In this case, it is recommended to sleep for 30s before consuming the next message, which will reduce the pressure on the broker to retry the message. RocketMQ will set up a retry queue named “%RETRY%+consumerGroup” for each consumer group(Note that the retry queue for this topic is for consumer groups, not for each topic) to temporarily save messages cannot be consumed by customer due to all kinds of reasons. Considering that it takes some time for the exception to recover, multiple retry levels are set for the retry queue, and each retry level has a corresponding re-deliver delay. The more retries, the greater the deliver delay. RocketMQ first save retry messages to the delay queue which topic is named “SCHEDULE_TOPIC_XXXX”, then background schedule task will save the messages to “%RETRY%+consumerGroup” retry queue according to their corresponding delay. -## Message Resend +## 10 Message Resend When a producer sends a message, the synchronous message will be resent if fails, the asynchronous message will retry and oneway message is without any guarantee. Message resend ensures that messages are sent successfully and without lost as much as possible, but it can lead to message duplication, which is an unavoidable problem in RocketMQ. Under normal circumstances, message duplication will not occur, but when there is a large number of messages and network jitter, message duplication will be a high-probability event. In addition, producer initiative messages resend and the consumer load changes will also result in duplicate messages. The message retry policy can be set as follows: - retryTimesWhenSendFailed: Synchronous message retry times when send failed, default value is 2, so the producer will try to send retryTimesWhenSendFailed + 1 times at most. To ensure that the message is not lost, producer will try sending the message to another broker instead of selecting the broker that failed last time. An exception will be thrown if it reaches the retry limit, and the client should guarantee that the message will not be lost. Messages will resend when RemotingException, MQClientException, and partial MQBrokerException occur. - retryTimesWhenSendAsyncFailed: Asynchronous message retry times when send failed, asynchronous retry sends message to the same broker instead of selecting another one and does not guarantee that the message wont lost. - retryAnotherBrokerWhenNotStoreOK: Message flush disk (master or slave) timeout or slave not available (return status is not SEND_OK), whether to try to send to another broker, default value is false. Very important messages can set to true. -## Flow Control +## 11 Flow Control Producer flow control, because broker processing capacity reaches a bottleneck; Consumer flow control, because the consumption capacity reaches a bottleneck. Producer flow control: @@ -80,7 +80,7 @@ Consumer flow control: The result of consumer flow control is to reduce the pull frequency. -## Dead Letter Queue +## 12 Dead Letter Queue Dead letter queue is used to deal messages that cannot be consumed normally. When a message is consumed failed at first time, the message queue will automatically resend the message. If the consumption still fails after the maximum number retry, it indicates that the consumer cannot properly consume the message under normal circumstances. At this time, the message queue will not immediately abandon the message, but send it to the special queue corresponding to the consumer. RocketMQ defines the messages that could not be consumed under normal circumstances as Dead-Letter Messages, and the special queue in which the Dead-Letter Messages are saved as Dead-Letter Queues. In RocketMQ, the consumer instance can consume again by resending messages in the Dead-Letter Queue using console. \ No newline at end of file From b5a9423751bbfa59978af256b1412e721a00dd7a Mon Sep 17 00:00:00 2001 From: Cloud_Leung <339330952@qq.com> Date: Sun, 3 Mar 2019 11:17:26 +0800 Subject: [PATCH 063/122] Update Design_Store.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 编号75:更新英文文档：Design_Store.md --- docs/en/Design_Store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Design_Store.md b/docs/en/Design_Store.md index 6cd85404b8..f701893c35 100644 --- a/docs/en/Design_Store.md +++ b/docs/en/Design_Store.md @@ -18,7 +18,7 @@ The message storage architecture diagram consists of 3 files related to message From the above architecture of the RocketMQ message storage, we can see RocketMQ uses a hybrid storage structure, that is, all the queues in an instance of the broker share a single log file `CommitLog` to store messages. RocketMQ's hybrid storage structure(messages of multiple topics are stored in one CommitLog) uses a separate storage structure for the data and index parts for Producer and Consumer respectively. The Producer sends the message to the Broker, then the Broker persists the message to the CommitLog file synchronously or asynchronously. As long as the message is persisted to the CommitLog on the disk, the message sent by the Producer will not be lost. Because of this, Consumer will definitely have the opportunity to consume this message. When no message can be pulled, the consumer can wait for the next pull. And the server also supports the long polling mode: if a pull request pulls no messages, the Broker can wait for 30 seconds, as long as new message arrives in this interval, it will be returned directly to the consumer. Here, RocketMQ's specific approach is using Broker's background service thread `ReputMessageService` to continuously dispatch requests and asynchronously build ConsumeQueue (Logical Queue) and IndexFile data. -## 2 PageCache and memory map +## 2 PageCache and Memory Map PageCache is a cache of files by the operating system to speed up the reading and writing of files. In general, the speed of sequential read and write files is almost the same as the speed of read and write memory. The main reason is that the OS uses a portion of the memory as PageCache to optimize the performance of the read and write operations. For data writing, the OS will first write to the Cache, and then the `pdflush` kernel thread asynchronously flush the data in the Cache to the physical disk. For data reading, if it can not hit the page cache when reading a file at a time, the OS will read the file from the physical disk and prefetch the data files of other neighboring blocks sequentially. From 44f8c11ae2ebd98e7cd8dbbf8f43004a3d9b9b07 Mon Sep 17 00:00:00 2001 From: Lunaticf <724922441@qq.com> Date: Sun, 3 Mar 2019 12:02:24 +0800 Subject: [PATCH 064/122] Modify English document for Example_Transaction.md --- docs/en/Example_Transaction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Example_Transaction.md b/docs/en/Example_Transaction.md index 77ee41d838..9dc700e375 100644 --- a/docs/en/Example_Transaction.md +++ b/docs/en/Example_Transaction.md @@ -1,4 +1,4 @@ -# Transaction message example +# Transaction Message Example ## 1 Transaction message status There are three states for transaction message: From 036a9ce7dd49247c5f47bf342b1c0c3246d65b19 Mon Sep 17 00:00:00 2001 From: zengqhb <35290210+zengqhb@users.noreply.github.com> Date: Sun, 3 Mar 2019 12:10:17 +0800 Subject: [PATCH 065/122] Update Operations_Producer.md Add a period at the end of the first paragraph --- docs/en/Operations_Producer.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Operations_Producer.md b/docs/en/Operations_Producer.md index 75248b16a3..68ef4a6921 100644 --- a/docs/en/Operations_Producer.md +++ b/docs/en/Operations_Producer.md @@ -2,7 +2,7 @@ ---- ##### 1 Message Sending Tips ###### 1.1 The Use of Tags -One application instance should use one topic as much as possible and the subtype of messages can be marked by tags. Tag provides extra flexibility to users. In the consume subscribing process, the messages filtering can only be handled by using tags when the tags are specified in the message sending process: `message.setTags("TagA")` +One application instance should use one topic as much as possible and the subtype of messages can be marked by tags. Tag provides extra flexibility to users. In the consume subscribing process, the messages filtering can only be handled by using tags when the tags are specified in the message sending process: `message.setTags("TagA")`. ###### 1.2 The Use of Keys A business key can be set in one message and it will be easier to look up the message on a broker server to diagnose issues during development. Each message will be created index(hash index) by server, instance can query the content of this message by topic and key and who consumes the message.Because of the hash index, make sure that the key should be unique in order to avoid potential hash index conflict. ``` java @@ -41,4 +41,4 @@ The message sending is usually a process like below: * Sever handles request * Sever returns response to client -The total costing time of sending one message is the sum of costing time of three steps above. Some situations demand that total costing time must be in a quite low level, however, do not take reliable performance into consideration, such as log collection. This kind of application could be called in one-way mode, which means client sends request but not wait for response. In this kind of mode, the cost of sending request is only a call of system operation which means one operation writing data to client socket buffer. Generally, the time cost of this process will be controlled n microseconds level. \ No newline at end of file +The total costing time of sending one message is the sum of costing time of three steps above. Some situations demand that total costing time must be in a quite low level, however, do not take reliable performance into consideration, such as log collection. This kind of application could be called in one-way mode, which means client sends request but not wait for response. In this kind of mode, the cost of sending request is only a call of system operation which means one operation writing data to client socket buffer. Generally, the time cost of this process will be controlled n microseconds level. From 161a4572b68c5b2fcea258a1ecbbf37053c4df99 Mon Sep 17 00:00:00 2001 From: Brain Date: Sun, 3 Mar 2019 12:20:25 +0800 Subject: [PATCH 066/122] Update Design_Filter.md --- docs/en/Design_Filter.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Design_Filter.md b/docs/en/Design_Filter.md index 246e162349..28eb59948e 100644 --- a/docs/en/Design_Filter.md +++ b/docs/en/Design_Filter.md @@ -5,6 +5,6 @@ RocketMQ - a distributed message queue, is different with all other MQ middlewar The RocketMQ has two mainly filter types: -* Tag filtering: Consumer can specify not only the message topic but also the message tag values, when subscribing. Multiple tag values need to be separated by '||'. When consumer subscribing a message, it builds the subscription request into a `SubscriptionData` object and sends a Pull message request to the Broker side. Before the Broker side reads data from the RocketMQ file storage layer - Store, it will construct a `MessageFilter` using the `SubscriptionData` object and then pass it to the Store. Store get a record from `ConsumeQueue`, and it will filter the message by the saved tag hashcode, it is unable to filter the messages exactly in the server side because of only the hashcode will be used when filtering, Therefore, after the Consumer pulls the message, it also needs to compare the original tag string of the message. If the original tag string is not same with the expected, the message will be ignored. +* Tag filtering: Consumer can specify not only the message topic but also the message tag values, when subscribing. Multiple tag values need to be separated by '||'. When consumer subscribing a message, it builds the subscription request into a `SubscriptionData` object and sends a pull message request to the Broker side. Before the Broker side reads data from the RocketMQ file storage layer - Store, it will construct a `MessageFilter` using the `SubscriptionData` object and then pass it to the Store. Store get a record from `ConsumeQueue`, and it will filter the message by the saved tag hashcode, it is unable to filter the messages exactly in the server side because of only the hashcode will be used when filtering, Therefore, after the Consumer pulls the message, it also needs to compare the original tag string of the message. If the original tag string is not same with the expected, the message will be ignored. -* SQL92 filtering: This filter behavior is almost same with the above Tag filtering method. The only difference is on the way how Store works. The rocketmq-filter module is responsible for the construction and execution of the real SQL expression. Executing an SQL expression every time a filter is executed affects efficiency, so RocketMQ uses BloomFilter to avoid doing it every time. The expression context of SQL92 is a property of the message. +* SQL92 filtering: This filter behavior is almost same with the above `Tag filtering` method. The only difference is on the way how Store works. The rocketmq-filter module is responsible for the construction and execution of the real SQL expression. Executing an SQL expression every time a filter is executed affects efficiency, so RocketMQ uses BloomFilter to avoid doing it every time. The expression context of SQL92 is a property of the message. From 2fb17d17131519a291661fdc068162f81f1e8251 Mon Sep 17 00:00:00 2001 From: djangogao Date: Sun, 3 Mar 2019 12:30:39 +0800 Subject: [PATCH 067/122] [RIP-76] Remove unnecessary dots for Design_Trancation.md --- docs/en/Design_Trancation.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/Design_Trancation.md b/docs/en/Design_Trancation.md index e5f10ed75d..e0d9c3103f 100644 --- a/docs/en/Design_Trancation.md +++ b/docs/en/Design_Trancation.md @@ -7,20 +7,20 @@ Apache RocketMQ supports distributed transaction message from version 4.3.0. Roc ### 1.1 The Process of RocketMQ Transaction Message The picture above shows the overall architecture of transaction message, including the sending of message(commit-request phase), the sending of commit/rollback(commit phase) and the compensation process. -1. The sending of message and Commit/Rollback. +1 The sending of message and Commit/Rollback. (1) Sending the message(named Half message in RocketMQ) (2) The server responds the writing result(success or failure) of Half message. (3) Handle local transaction according to the result(local transaction won't be executed when the result is failure). (4) Sending Commit/Rollback to broker according to the result of local transaction(Commit will generate message index and make the message visible to consumers). -2. Compensation process +2 Compensation process (1) For a transaction message without a Commit/Rollback (means the message in the pending status), a "back-check" request is initiated from the broker. (2) The Producer receives the "back-check" request and checks the status of the local transaction corresponding to the "back-check" message. (3) Redo Commit or Rollback based on local transaction status. The compensation phase is used to resolve the timeout or failure case of the message Commit or Rollback. ### 1.2 The design of RocketMQ Transaction Message -1. Transaction message is invisible to users in first phase(commit-request phase) +1 Transaction message is invisible to users in first phase(commit-request phase) Upon on the main process of transaction message, the message of first phase is invisible to the user. This is also the biggest difference from normal message. So how do we write the message while making it invisible to the user? And below is the solution of RocketMQ: if the message is a Half message, the topic and queueId of the original message will be backed up, and then changes the topic to RMQ_SYS_TRANS_HALF_TOPIC. Since the consumer group does not subscribe to the topic, the consumer cannot consume the Half message. Then RocketMQ starts a timing task, pulls the message for RMQ_SYS_TRANS_HALF_TOPIC, obtains a channel according to producer group and sends a back-check to query local transaction status, and decide whether to submit or roll back the message according to the status. @@ -30,21 +30,21 @@ The compensation phase is used to resolve the timeout or failure case of the mes The specific implementation strategy of RocketMQ is: if the transaction message is written, topic and queueId of the message are replaced, and the original topic and queueId are stored in the properties of the message. Because the replace of the topic, the message will not be forwarded to the Consumer Queue of the original topic, and the consumer cannot perceive the existence of the message and will not consume it. In fact, changing the topic is the conventional method of RocketMQ(just recall the implementation mechanism of the delay message). -2. Commit/Rollback operation and introduction of Op message +2 Commit/Rollback operation and introduction of Op message After finishing writing a message that is invisible to the user in the first phase, here comes two cases in the second phase. One is Commit operation, after which the message needs to be visible to the user; the other one is Rollback operation, after which the first phase message(Half message) needs to be revoked. For the case of Rollback, since first-phase message itself is invisible to the user, there is no need to actually revoke the message (in fact, RocketMQ can't actually delete a message because it is a sequential-write file). But still some operation needs to be done to identity the final status of the message, to differ it from pending status message. To do this, the concept of "Op message" is introduced, which means the message has a certain status(Commit or Rollback). If a transaction message does not have a corresponding Op message, the status of the transaction is still undetermined (probably the second-phase failed). By introducing the Op message, the RocketMQ records an Op message for every Half message regardless it is Commit or Rollback. The only difference between Commit and Rollback is that when it comes to Commit, the index of the Half message is created before the Op message is written. -3. How Op message stored and the correspondence between Op message and Half message +3 How Op message stored and the correspondence between Op message and Half message RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation. ![](../cn/image/rocketmq_design_12.png) -4. Index construction of Half messages +4 Index construction of Half messages When performing Commit operation of the second phase, the index of the Half message needs to be built. Since the Half message is written to a special topic(RMQ_SYS_TRANS_HALF_TOPIC) in the first phase of 2PC, so it needs to be read out from the special topic when building index, and replace the topic and queueId with the real target topic and queueId, and then write through a normal message that is visible to the user. Therefore, in conclusion, the second phase recovers a complete normal message using the content of the Half message stored in the first phase, and then goes through the message-writing process. -5. How to handle the message failed in the second phase？ +5 How to handle the message failed in the second phase？ If commit/rollback phase fails, for example, a network problem causes the Commit to fail when you do Commit. Then certain strategy is required to make sure the message finally commit. RocketMQ uses a compensation mechanism called "back-check". The broker initiates a back-check request for the message in pending status, and sends the request to the corresponding producer side (the same producer group as the producer group who sent the Half message). The producer checks the status of local transaction and redo Commit or Rollback. The broker performs the back-check by comparing the RMQ_SYS_TRANS_HALF_TOPIC messages and the RMQ_SYS_TRANS_OP_HALF_TOPIC messages and advances the checkpoint(recording those transaction messages that the status are certain). From 65fe98bba43b14c23b740883c89358e05bad7f63 Mon Sep 17 00:00:00 2001 From: fengjian Date: Sun, 3 Mar 2019 12:49:40 +0800 Subject: [PATCH 068/122] Modify English document --- docs/en/Configuration_System.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..941dfa70d0 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -1,4 +1,4 @@ -# ** The system configuration** # +# The system configuration # This section focuses on the configuration of the system (JVM/OS) @@ -22,7 +22,7 @@ As for garbage collection, G1 collector with JDK 1.8 is recommended: -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -These GC options looks a little aggressive, but it’s proved to have good performance in our production environment +These GC options looks a little aggressive, but it’s proved to have good performance in our production environment. Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.So use rolling GC log file is recommended: From 8a585ced86a9725f5a8b20cbf4b08d2fe820c9e0 Mon Sep 17 00:00:00 2001 From: Miracles520 <1040325894@qq.com> Date: Sun, 3 Mar 2019 12:54:57 +0800 Subject: [PATCH 069/122] update Design_loadBlancing.md --- docs/en/Design_LoadBlancing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index b93b6ea86f..e6bb66f8ac 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,11 +1,11 @@ -## 4 Load Balancing +## Load Balancing Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. -### 4.1 Producer Load Balancing +### 1 Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 4.2 Consumer Load Balancing +### 2 Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. 1， Heartbeat Packet Sending on Consumer side From 8b060eac47572093d6137d682fccf683349a702c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9B=B9=E6=B5=A9?= Date: Sun, 3 Mar 2019 13:27:44 +0800 Subject: [PATCH 070/122] remove "." behind the number in each title --- docs/en/Troubleshoopting.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/Troubleshoopting.md b/docs/en/Troubleshoopting.md index d43d00e392..02a245ed04 100644 --- a/docs/en/Troubleshoopting.md +++ b/docs/en/Troubleshoopting.md @@ -1,6 +1,6 @@ # Operation FAQ -## 1. RocketMQ's mqadmin command error. +## 1 RocketMQ's mqadmin command error. > Problem: Sometimes after deploying the RocketMQ cluster, when you try to execute some commands of "mqadmin", the following exception will appear： > @@ -10,7 +10,7 @@ Solution: Execute `export NAMESRV_ADDR=ip:9876` (ip refers to the address of NameServer deployed in the cluster) on the VM that deploys the RocketMQ cluster.Then you will execute commands of "mqadmin" successfully. -## 2. The inconsistent version of RocketMQ between the producer and consumer leads to the problem that message can't be consumed normally. +## 2 The inconsistent version of RocketMQ between the producer and consumer leads to the problem that message can't be consumed normally. > Problem: The same producer sends a message, consumer A can consume, but consumer B can't consume, and the RocketMQ Console appears: > @@ -20,7 +20,7 @@ Solution: Execute `export NAMESRV_ADDR=ip:9876` (ip refers to the address of Na Solution: The jar package of RocketMQ, such as rocketmq-client, should be the same version on the consumer and producer. -## 3. When adding a new topic consumer group, historical messages can't be consumed. +## 3 When adding a new topic consumer group, historical messages can't be consumed. > Problem: When a new consumer group of the same topic is started, the consumed message is the current offset message, and the historical message is not obtained. @@ -50,11 +50,11 @@ consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_TIMESTAMP); ``` -## 4. How to enable reading data from Slave +## 4 How to enable reading data from Slave In some cases, the Consumer needs to reset the consume position to 1-2 days ago. At this time, on the Master Broker with limited memory, the CommitLog will carry a relatively heavy IO pressure, affecting the reading and writing of other messages on that Broker. You can enable `slaveReadEnable=true`. When Master Broker finds that the difference between the Consumer's consume position and the latest value of CommitLog exceeds the percentage of machine's memory (`accessMessageInMemoryMaxRatio=40%`), it will recommend Consumer to read from Slave Broker and relieve Master Broker's IO. -## 5. Performance +## 5 Performance Asynchronous flush disk is recommended to use spin lock. @@ -64,7 +64,7 @@ Asynchronous flush disk is recommended to open `TransientStorePoolEnable` and cl Synchronous flush disk is recommended to increase the `sendMessageThreadPoolNums` appropriately. The specific configuration needs to be tested. -## 6. The meaning and difference between msgId and offsetMsgId in RocketMQ +## 6 The meaning and difference between msgId and offsetMsgId in RocketMQ After sending message with RocketMQ, you will usually see the following log: From d45330c3854390b69b57779086434f7375420cde Mon Sep 17 00:00:00 2001 From: blackcat <18200135187@163.com> Date: Sun, 3 Mar 2019 14:23:15 +0800 Subject: [PATCH 071/122] [RIP-9] update the title and delete cluster setup --- docs/en/Deployment.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index 5d74c163d5..3a810974c3 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -1,6 +1,4 @@ -# Deployment Architectures and Setup Steps - -## Cluster Setup +# Installation Guides ### 1 Single Master mode From c644ed25409f347241bc37c67e27c8ec020543f4 Mon Sep 17 00:00:00 2001 From: hellozepp Date: Sun, 3 Mar 2019 14:25:52 +0800 Subject: [PATCH 072/122] Update Feature.md in RocketMQ/docs/en --- docs/en/Feature.md | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/docs/en/Feature.md b/docs/en/Feature.md index eb6a328f49..806d648291 100644 --- a/docs/en/Feature.md +++ b/docs/en/Feature.md @@ -1,8 +1,8 @@ # Features -## Subscribe and Publish +## 1 Subscribe and Publish Message publication refers to that a producer sends messages to a topic; Message subscription means a consumer follows a topic with certain tags and then consumes data from that topic. -## Message Ordering +## 2 Message Ordering Message ordering refers to that a group of messages can be consumed orderly as they are published. For example, an order generates three messages: order creation, order payment, and order completion. It only makes sense to consume them in their generated order, but orders can be consumed in parallel at the same time. RocketMQ can strictly guarantee these messages are in order. Orderly message are divided into global orderly message and partitioned orderly message. Global order means that all messages under a certain topic must be in order, partitioned order only requires each group of messages are consumed orderly. @@ -13,32 +13,32 @@ Applicable scenario: the performance requirement is not high, and all messages a For a given Topic, all messages are partitioned according to sharding key. Messages within the same partition are published and consumed in strict FIFO order. Sharding key is the key field to distinguish message's partition, which is a completely different concept from the key of ordinary messages. Applicable scenario: high performance requirement, with sharding key as the partition field, messages within the same partition are published and consumed according to FIFO principle strictly. -## Message Filter +## 3 Message Filter Consumers of RocketMQ can filter messages based on tags as well as support for user-defined attribute filtering. Message filter is currently implemented on the Broker side, with the advantage of reducing the network transmission of useless messages for Consumer and the disadvantage of increasing the burden on the Broker and relatively complex implementation. -## Message Reliability +## 4 Message Reliability RocketMQ supports high reliability of messages in several situations: -1) Broker shutdown normally -2) Broker abnormal crash -3) OS Crash -4) The machine is out of power, but it can be recovered immediately -5) The machine cannot be started up (the CPU, motherboard, memory and other key equipment may be damaged) -6) Disk equipment damaged +1 Broker shutdown normally +2 Broker abnormal crash +3 OS Crash +4 The machine is out of power, but it can be recovered immediately +5 The machine cannot be started up (the CPU, motherboard, memory and other key equipment may be damaged) +6 Disk equipment damaged In the four cases of 1), 2), 3), and 4) where the hardware resource can be recovered immediately, RocketMQ guarantees that the message will not be lost or a small amount of data will be lost (depending on whether the flush disk type is synchronous or asynchronous) 5) and 6) are single point of failure and cannot be recovered. Once it happens, all messages on the single point will be lost. In both cases, RocketMQ ensures that 99% of the messages are not lost through asynchronous replication, but a very few number of messages may still be lost. Synchronous double write mode can completely avoid single point of failure, which will surely affect the performance and suitable for the occasion of high demand for message reliability, such as money related applications. Note: RocketMQ supports synchronous double writes since version 3.0. -## At Least Once +## 5 At Least Once At least Once refers to that every message will be delivered at least once. RocketMQ supports this feature because the Consumer pulls the message locally and does not send an ack back to the server until it has consumed it. -## Backtracking Consumption +## 6 Backtracking Consumption Backtracking consumption refers to that the Consumer has consumed the message successfully, but the business needs to consume again. To support this function, the message still needs to be retained after the Broker sends the message to the Consumer successfully. The re-consumption is normally based on time dimension. For example, after the recovery of the Consumer system failure, the data one hour ago needs to be re-consumed, then the Broker needs to provide a mechanism to reverse the consumption progress according to the time dimension. RocketMQ supports backtracking consumption by time trace, with the time dimension down to milliseconds. -## Transactional Message +## 7 Transactional Message RocketMQ transactional message refers to the fact that the application of a local transaction and the sending of a Message operation can be defined in a global transaction which means both succeed or fail simultaneously. RocketMQ transactional message provides distributed transaction functionality similar to X/Open XA, enabling the ultimate consistency of distributed transactions through transactional message. -## Scheduled Message +## 8 Scheduled Message Scheduled message(delay queue) refers to that messages are not consumed immediately after they are sent to the broker, but waiting to be delivered to the real topic after a specific time. The broker has a configuration item, messageDelayLevel, with default values “1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h”, 18 levels. Users can configure a custom messageDelayLevel. Note that messageDelayLevel is a broker's property rather than a topic's. When sending a message, just set the delayLevel level: msg.setDelayLevel(level). There are three types of levels: @@ -50,21 +50,21 @@ Scheduled messages are temporarily saved in a topic named SCHEDULE_TOPIC_XXXX, a Note that Scheduled messages are counted both the first time they are written and the time they are scheduled to be written to the real topic, so both the send number and the TPS are increased. -## Message Retry +## 9 Message Retry When the Consumer fails to consume the message, a retry mechanism is needed to make the message to be consumed again. Consumer's consume failure can usually be classified as follows: - due to the reasons of the message itself, such as deserialization failure, the message data itself cannot be processed (for example, the phone number of the current message is cancelled and cannot be charged), etc. This kind of error usually requires skipping this message and consuming others since immediately retry would be failed 99%, so it is better to provide a timed retry mechanism that retries after 10 seconds. - due to the reasons of dependent downstream application services are not available, such as db connection is not usable, perimeter network is not unreachable, etc. When this kind of error is encountered, consuming other messages will also result in an error even if the current failed message is skipped. In this case, it is recommended to sleep for 30s before consuming the next message, which will reduce the pressure on the broker to retry the message. RocketMQ will set up a retry queue named “%RETRY%+consumerGroup” for each consumer group(Note that the retry queue for this topic is for consumer groups, not for each topic) to temporarily save messages cannot be consumed by customer due to all kinds of reasons. Considering that it takes some time for the exception to recover, multiple retry levels are set for the retry queue, and each retry level has a corresponding re-deliver delay. The more retries, the greater the deliver delay. RocketMQ first save retry messages to the delay queue which topic is named “SCHEDULE_TOPIC_XXXX”, then background schedule task will save the messages to “%RETRY%+consumerGroup” retry queue according to their corresponding delay. -## Message Resend +## 10 Message Resend When a producer sends a message, the synchronous message will be resent if fails, the asynchronous message will retry and oneway message is without any guarantee. Message resend ensures that messages are sent successfully and without lost as much as possible, but it can lead to message duplication, which is an unavoidable problem in RocketMQ. Under normal circumstances, message duplication will not occur, but when there is a large number of messages and network jitter, message duplication will be a high-probability event. In addition, producer initiative messages resend and the consumer load changes will also result in duplicate messages. The message retry policy can be set as follows: - retryTimesWhenSendFailed: Synchronous message retry times when send failed, default value is 2, so the producer will try to send retryTimesWhenSendFailed + 1 times at most. To ensure that the message is not lost, producer will try sending the message to another broker instead of selecting the broker that failed last time. An exception will be thrown if it reaches the retry limit, and the client should guarantee that the message will not be lost. Messages will resend when RemotingException, MQClientException, and partial MQBrokerException occur. - retryTimesWhenSendAsyncFailed: Asynchronous message retry times when send failed, asynchronous retry sends message to the same broker instead of selecting another one and does not guarantee that the message wont lost. - retryAnotherBrokerWhenNotStoreOK: Message flush disk (master or slave) timeout or slave not available (return status is not SEND_OK), whether to try to send to another broker, default value is false. Very important messages can set to true. -## Flow Control +## 11 Flow Control Producer flow control, because broker processing capacity reaches a bottleneck; Consumer flow control, because the consumption capacity reaches a bottleneck. Producer flow control: @@ -80,7 +80,7 @@ Consumer flow control: The result of consumer flow control is to reduce the pull frequency. -## Dead Letter Queue +## 12 Dead Letter Queue Dead letter queue is used to deal messages that cannot be consumed normally. When a message is consumed failed at first time, the message queue will automatically resend the message. If the consumption still fails after the maximum number retry, it indicates that the consumer cannot properly consume the message under normal circumstances. At this time, the message queue will not immediately abandon the message, but send it to the special queue corresponding to the consumer. RocketMQ defines the messages that could not be consumed under normal circumstances as Dead-Letter Messages, and the special queue in which the Dead-Letter Messages are saved as Dead-Letter Queues. In RocketMQ, the consumer instance can consume again by resending messages in the Dead-Letter Queue using console. \ No newline at end of file From 240357b8cc059ede900e634da5e54ceb1279802d Mon Sep 17 00:00:00 2001 From: shudo1219 Date: Sun, 3 Mar 2019 14:55:17 +0800 Subject: [PATCH 073/122] [RIP-9] update Concept.md in RocketMQ docs/en --- docs/en/Concept.md | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/docs/en/Concept.md b/docs/en/Concept.md index 84aaa7c42a..066165ed82 100644 --- a/docs/en/Concept.md +++ b/docs/en/Concept.md @@ -2,40 +2,40 @@ ## 1 Message Model -The RocketMQ message model is mainly composed of Producer, Broker and Consumer. The Producer is responsible for producing messages, the Consumer is responsible for consuming messages, and the Broker is responsible for storing messages. -The Broker corresponds to one server during actual deployment, and each Broker can store messages from multiple topics, and messages from each Topic can be stored in a different Broker by sharding strategy. -The Message Queue is used to store physical offsets of messages, and the Message addresses in multiple Topic are stored in multiple Message queues. The Consumer Group consists of multiple Consumer instances. +RocketMQ message model is mainly composed of Producer, Broker and Consumer. The producer is responsible for producing messages and the consumer is for consuming messages, while the broker stores messages. +The broker is an independent server during actual deployment, and each broker can store messages from multiple topics. Even messages from the same topic can be stored in the different brokers by sharding strategy. +The message queue is used to store physical offsets of messages, and the message addresses are stored in seperate queues. The consumer group consists of multiple consumer instances. ## 2 Producer -The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the business application systems to brokers. The RocketMQ provides multiple paradigms of sending: synchronous, asynchronous,sequential and one-way. Both synchronous and asynchronous methods require the Broker to return confirmation information, while one-way sending is not required. +The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the systems to brokers. RocketMQ provides multiple paradigms of sending: synchronous, asynchronous, sequential and one-way. Both synchronous and asynchronous methods require the confirmation information return from the Broker, but one-way don't need. ## 3 Consumer - The Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The Consumer pulls messages from brokers and feeds them into application. In perspective of user application, two types of consumers are provided:Pull Consumer and Push Consumer. +The Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The consumer pulls messages from brokers and feeds them into application. From the perspective of user, two types of consumers are provided: pull consumer and push consumer. ## 4 Topic -The Topic refers to a collection of one kind message. Each topic contains several messages and one message can only belong to one topic. The Topic is the basic unit of RocketMQ for message subscription. +The Topic refers to a collection of one kind of message. Each topic contains several messages and one message can only belong to one topic. The topic is the basic unit of RocketMQ for message subscription. ## 5 Broker Server -As the role of the transfer station, the Broker Server stores, and forwards messages. In the RocketMQ system, the Broker Server is responsible for receiving messages sent from producers, storing them and preparing to handle pull requests. It also stores message related meta data, including consumer groups, consuming progress, topics and queues info. +As the role of the transfer station, the Broker Server stores and forwards messages. In RocketMQ, the broker server is responsible for receiving messages sent from producers, storing them and preparing to handle pull requests. It also stores the related message meta data, including consumer groups, consuming progress, topics, queues info and so on. ## 6 Name Server -The Name Server serves as the provider of routing service. The Producer or Consumer can find the list of Broker IP addresses corresponding each topic through name server. Multiple Name Servers can be deployed as a cluster, but are independent of each other and do not exchange information. +The Name Server serves as the provider of routing service. The producer or the consumer can find the list of broker IP addresses for each topic through name server. Multiple name servers can be deployed in one cluster, but they are independent of each other and do not exchange information. ## 7 Pull Consumer - A type of Consumer. Applications are usually pulls messages from brokers by actively calling the Consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once batches of messages are pulled, user application initiates consuming process. +A type of Consumer. The application pulls messages from brokers by actively invoking the consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process. ## 8 Push Consumer - A type of Consumer. Under this mode, after the Broker receives the data, it will actively push it to the consumer, which is generally of high real-time performance. +A type of Consumer. Under this high real-time performance mode, it will push the message to the consumer actively when the Broker receives the data. ## 9 Producer Group - A collection of the same type of Producer, which sends the same type of messages with consistent logic. If a transaction message is sent and the original producer crashes after sending, the Broker Server will contacts other Producer in the same Producer group to commit or rollback the transactional message. +A collection of the same type of Producer, which sends the same type of messages with consistent logic. If a transaction message is sent and the original producer crashes after sending, the broker server will contact other producers in the same producer group to commit or rollback the transactional message. ## 10 Consumer Group - A collection of the same type of Consumer, which sends the same type of messages with consistent logic. The Consumer Group makes load-balance and fault-tolerance super easy in terms of message consuming. -Warning: consumer instances of one consumer group must have exactly the same topic subscription(s). +A collection of the same type of Consumer, which sends the same type of messages with consistent logic. The consumer group makes load-balance and fault-tolerance super easy in terms of message consuming. +Warning: consumer instances of one consumer group must have exactly the same topic subscription(s). -RocketMQ supports two type of consumption mode:Clustering and Broadcasting. +RocketMQ supports two types of consumption mode:Clustering and Broadcasting. ## 11 Consumption Mode - Clustering -Under the Clustering mode, all messages from one topic will be delivered to all consumer instances averagely as much as possible. That is, one message can be consumed by only one consumer instance. +Under the Clustering mode, all the messages from one topic will be delivered to all the consumers instances averagely as much as possible. That is, one message can be consumed by only one consumer instance. ## 12 Consumption Mode - Broadcasting -Under the Broadcasting mode, each Consumer instance of the same Consumer Group receives every message that published to the corresponding topic. +Under the Broadcasting mode, each consumer instance of the same consumer group receives every message published to the corresponding topic. ## 13 Normal Ordered Message -Under the Normal Ordered Message mode, the messages received by consumers from the same ConsumeQueue are sequential, while the messages received from different message queues may be non-sequential. +Under the Normal Ordered Message mode, the messages received by consumers from the same ConsumeQueue are sequential, but the messages received from the different message queues may be non-sequential. ## 14 Strictly Ordered Message -Under the Strictly Ordered Message mode, all messages received by the consumers from the same topic are sequential, as the order they were stored. +Under the Strictly Ordered Message mode, all messages received by the consumers from the same topic are sequential as the order they are stored. ## 15 Message -The physical carrier of information transmitted by a messaging system, the smallest unit of production and consumption data, each message must belong to a topic. -Each Message in RocketMQ has a unique Message ID and can carry a key, generally used to store business-related value. The system provides the function to query messages by Message ID and Key. +The physical carrier of information transmitted by a messaging system, the smallest unit of production and consumption data, each message must belong to one topic. +Each Message in RocketMQ has a unique message id and can carry a key used to store business-related value. The system has the function to query messages by its id or key. ## 16 Tag - Flags set for messages to distinguish different types of messages under the same topic, functioning as a "sub-topic". Messages from the same business unit can set different tags under the same topic in terms of different business purposes. The Tag can effectively maintain the clarity and consistency of the code and optimize the query system provided by RocketMQ. The Consumer can realize different "sub-topic" by using Tag, so as to achieve better expansibility. +Flags set for messages to distinguish different types of messages under the same topic, functioning as a "sub-topic". Messages from the same business unit can set different tags under the same topic in terms of different business purposes. The tag can effectively maintain the clarity and consistency of the code and optimize the query system provided by RocketMQ. The consumer can realize different "sub-topic" by using tag in order to achieve better expansibility. From f8b24a7ecb6e92032323cba7baaebb5186db0697 Mon Sep 17 00:00:00 2001 From: shudo1219 Date: Sun, 3 Mar 2019 15:26:21 +0800 Subject: [PATCH 074/122] [RIP-9] update Concept.md and change '.' to ',' in RocketMQ docs/en --- docs/en/Concept.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Concept.md b/docs/en/Concept.md index 066165ed82..e1e5ecbb42 100644 --- a/docs/en/Concept.md +++ b/docs/en/Concept.md @@ -16,7 +16,7 @@ As the role of the transfer station, the Broker Server stores and forwards messa ## 6 Name Server The Name Server serves as the provider of routing service. The producer or the consumer can find the list of broker IP addresses for each topic through name server. Multiple name servers can be deployed in one cluster, but they are independent of each other and do not exchange information. ## 7 Pull Consumer -A type of Consumer. The application pulls messages from brokers by actively invoking the consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process. +A type of Consumer, the application pulls messages from brokers by actively invoking the consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process. ## 8 Push Consumer A type of Consumer. Under this high real-time performance mode, it will push the message to the consumer actively when the Broker receives the data. ## 9 Producer Group From b0d12ce0980bf6863d10a8a0a53ebe6fc9b60535 Mon Sep 17 00:00:00 2001 From: zhangjianwei <“CSSVDD@gmail.com”> Date: Sun, 3 Mar 2019 15:26:47 +0800 Subject: [PATCH 075/122] update rocketmq/docs/en/Example_Delay.md, delete some punctuation character --- docs/en/Example_Delay.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/Example_Delay.md b/docs/en/Example_Delay.md index 80785256d9..4de4b74749 100644 --- a/docs/en/Example_Delay.md +++ b/docs/en/Example_Delay.md @@ -1,6 +1,6 @@ # Schedule example -### 1.Start consumer to wait for incoming subscribed messages +### 1 Start consumer to wait for incoming subscribed messages ```java import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; @@ -35,7 +35,7 @@ public class ScheduledMessageConsumer { } ``` -### 2.Send scheduled messages +### 2 Send scheduled messages ```java import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -64,15 +64,15 @@ public class ScheduledMessageProducer { } ``` -### 3.Verification +### 3 Verification You should see messages are consumed about 10 seconds later than their storing time. -### 4.Use scenarios for scheduled messages +### 4 Use scenarios for scheduled messages For example, in e-commerce, if an order is submitted, a delay message can be sent, and the status of the order can be checked after 1 hour. If the order is still unpaid, the order can be cancelled and the inventory released. -### 5.Restrictions on the use of scheduled messages +### 5 Restrictions on the use of scheduled messages ```java // org/apache/rocketmq/store/config/MessageStoreConfig.java From 103aaa854e63c4a1eb66c138060df2af8a2b50fc Mon Sep 17 00:00:00 2001 From: dust6174 Date: Sun, 3 Mar 2019 15:35:24 +0800 Subject: [PATCH 076/122] Delete the Introduction and underline --- docs/en/Operations_Trace.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/en/Operations_Trace.md b/docs/en/Operations_Trace.md index b6c5d45620..6a7ea8c716 100644 --- a/docs/en/Operations_Trace.md +++ b/docs/en/Operations_Trace.md @@ -1,8 +1,6 @@ # Message Trace -## Introduction -This document focuses on how to quickly deploy and use RocketMQ clusters that support message trace features ## 1. Key Attributes of Message Trace Data From d7dd2a8dbbdc754741840d25a380d5a2623c3231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E8=89=AF=E5=9B=BD?= Date: Sun, 3 Mar 2019 15:38:18 +0800 Subject: [PATCH 077/122] [RIP-78] update Example_Filter.md in RocketMQ docs/en --- docs/en/Example_Filter.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/Example_Filter.md b/docs/en/Example_Filter.md index 69669a1bdf..caf34cb7d5 100644 --- a/docs/en/Example_Filter.md +++ b/docs/en/Example_Filter.md @@ -1,7 +1,7 @@ # Filter Example ---------- -In most cases, tag is a simple and useful design to select message you want. For example: +In most cases, tag is a simple and useful design to select messages you want. For example: ```java DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_EXAMPLE"); @@ -28,7 +28,7 @@ SQL feature could do some calculation through the properties you put in when sen ------------ ``` -## 1.Grammars +## 1 Grammars RocketMQ only defines some basic grammars to support this feature. You could also extend it easily. - Numeric comparison, like **>**, **>=**, **<**, **<=**, **BETWEEN**, **=**; @@ -43,13 +43,13 @@ Constant types are: - **NULL**, special constant; - Boolean, **TRUE** or **FALSE**; -## 2.Usage constraints +## 2 Usage constraints Only push consumer could select messages by SQL92. The interface is: ``` public void subscribe(finalString topic, final MessageSelector messageSelector) ``` -## 3.Producer example +## 3 Producer example You can put properties in message through method putUserProperty when sending. ```java @@ -67,7 +67,7 @@ producer.shutdown(); ``` -## 4.Consumer example +## 4 Consumer example Use `MessageSelector.bySql` to select messages through SQL when consuming. From 02bf9081ee260f7e6abc9268e7576d370043a562 Mon Sep 17 00:00:00 2001 From: dust6174 Date: Sun, 3 Mar 2019 15:48:45 +0800 Subject: [PATCH 078/122] [RIP-9]Delete the Introduction and underlines --- docs/en/Operations_Trace.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/Operations_Trace.md b/docs/en/Operations_Trace.md index 6a7ea8c716..5ac16a6533 100644 --- a/docs/en/Operations_Trace.md +++ b/docs/en/Operations_Trace.md @@ -1,7 +1,5 @@ # Message Trace - - ## 1. Key Attributes of Message Trace Data | Producer | Consumer | Broker | @@ -103,4 +101,5 @@ The initialization of `DefaultMQProducer` and `DefaultMQPushConsumer` instances DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true,"Topic_test11111"); ...... + ``` \ No newline at end of file From 67182a40941fa97fe536afdcf0b0c577229ad7e9 Mon Sep 17 00:00:00 2001 From: HollisGW Date: Sun, 3 Mar 2019 15:49:00 +0800 Subject: [PATCH 079/122] Update Example_Orderly.md --- docs/en/Example_Orderly.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Example_Orderly.md b/docs/en/Example_Orderly.md index 8ae41f526f..e9cde37491 100644 --- a/docs/en/Example_Orderly.md +++ b/docs/en/Example_Orderly.md @@ -1,4 +1,4 @@ -# Example for ordered messages +# Example for Ordered Messages RocketMQ provides ordered messages using FIFO order. All related messages need to be sent into the same message queue in an orderly manner. @@ -168,7 +168,7 @@ public class Producer { ``` -## 2.2 Consume ordered messages +## 2 Consume ordered messages ```java From 70c88dd37e3a505121f7d10363e09703e2e3c6b5 Mon Sep 17 00:00:00 2001 From: Alice-moon Date: Sun, 3 Mar 2019 15:49:48 +0800 Subject: [PATCH 080/122] Update Design_Remoting.md --- docs/en/Design_Remoting.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en/Design_Remoting.md b/docs/en/Design_Remoting.md index ff2c99d2e6..4ce884fdf5 100644 --- a/docs/en/Design_Remoting.md +++ b/docs/en/Design_Remoting.md @@ -1,4 +1,4 @@ -## 2 Communication Mechanism + RocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows: (1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds. (2) When message producer Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. @@ -9,9 +9,9 @@ From 1) ~ 3 above, we can see that both Producer, Broker and NameServer communic rocketmq-remoting module is the module responsible for network communication in RocketMQ message queue. It is relied on and referenced by almost all other modules (such as rocketmq-client,rocketmq-broker,rocketmq-namesrv) that need network communication. In order to realize the efficient data request and reception between the client and the server, the RocketMQ message queue defines the communication protocol and extends the communication module on the basis of Netty. -### 2.1 Remoting Communication Class Structure +### 1 Remoting Communication Class Structure ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_3.png) -### 2.2 Protocol Design and Codec +### 2 Protocol Design and Code When a message is sent between Client and Server, a protocol convention is needed for the message sent, so it is necessary to customize the message protocol of RocketMQ. At the same time, in order to efficiently transmit messages and read the received messages, it is necessary to encode and decode the messages. In RocketMQ, the RemotingCommand class encapsulates all data content in the process of message transmission, which includes not only all data structures, but also encoding and decoding operations. Header field | Type | Request desc | Response desc @@ -33,10 +33,10 @@ From the above figure, the transport content can be divided into four parts: (3) Header data: serialized header data; (4) Message body data: binary byte data content of message body; -#### 2.3 Message Communication Mode and Procedure +#### 3 Message Communication Mode and Procedure There are three main ways to support communication in RocketMQ message queue: synchronous (sync), asynchronous (async), one-way (oneway). The "one-way" communication mode is relatively simple and is generally used in sending heartbeat packets without paying attention to its Response. Here, mainly introduce the asynchronous communication flow of RocketMQ. ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_5.png) -#### 2.4 Reactor Multithread Design +#### 4 Reactor Multithread Design The RPC communication of RocketMQ uses Netty component as the underlying communication library, and also follows the Reactor multithread model. At the same time, some extensions and optimizations are made on it. ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_6.png) Above block diagram can roughly understand the Reactor multi-thread model of NettyRemotingServer in RocketMQ. A Reactor main thread (eventLoopGroupBoss, is 1 above) is responsible for listening to TCP network connection requests, establishing connections, creating SocketChannel, and registering on selector. The source code of RocketMQ automatically selects NIO and Epoll according to the type of OS. Then listen to real network data. After you get the network data, you throw it to the Worker thread pool (eventLoopGroupSelector, is the "N" above, the default is 3 in the source code). You need to do SSL verification, codec, idle check, network connection management before you really execute the business logic. These tasks to defaultEventExecutorGroup (that is, "M1" above, the default set to 8 in the source code) to do. The processing business operations are executed in the business thread pool. According to the RomotingCommand business request code, the corresponding processor is found in the processorTable local cache variable and encapsulated into the task, and then submitted to the corresponding business processor processing thread pool for execution (sendMessageExecutor,). Take sending a message, for example, the "M2" above. The thread pool continues to increase in several steps from entry to business logic, which is related to the complexity of each step. The more complex the thread pool is, the wider the concurrent channel is required. From efa7798443b22f6581176380ab9fccaf0feae7b4 Mon Sep 17 00:00:00 2001 From: Alice-moon Date: Sun, 3 Mar 2019 15:50:54 +0800 Subject: [PATCH 081/122] Update Design_Remoting.md --- docs/en/Design_Remoting.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Design_Remoting.md b/docs/en/Design_Remoting.md index 4ce884fdf5..4f8a4b5ae6 100644 --- a/docs/en/Design_Remoting.md +++ b/docs/en/Design_Remoting.md @@ -33,10 +33,10 @@ From the above figure, the transport content can be divided into four parts: (3) Header data: serialized header data; (4) Message body data: binary byte data content of message body; -#### 3 Message Communication Mode and Procedure +### 3 Message Communication Mode and Procedure There are three main ways to support communication in RocketMQ message queue: synchronous (sync), asynchronous (async), one-way (oneway). The "one-way" communication mode is relatively simple and is generally used in sending heartbeat packets without paying attention to its Response. Here, mainly introduce the asynchronous communication flow of RocketMQ. ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_5.png) -#### 4 Reactor Multithread Design +### 4 Reactor Multithread Design The RPC communication of RocketMQ uses Netty component as the underlying communication library, and also follows the Reactor multithread model. At the same time, some extensions and optimizations are made on it. ![](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_6.png) Above block diagram can roughly understand the Reactor multi-thread model of NettyRemotingServer in RocketMQ. A Reactor main thread (eventLoopGroupBoss, is 1 above) is responsible for listening to TCP network connection requests, establishing connections, creating SocketChannel, and registering on selector. The source code of RocketMQ automatically selects NIO and Epoll according to the type of OS. Then listen to real network data. After you get the network data, you throw it to the Worker thread pool (eventLoopGroupSelector, is the "N" above, the default is 3 in the source code). You need to do SSL verification, codec, idle check, network connection management before you really execute the business logic. These tasks to defaultEventExecutorGroup (that is, "M1" above, the default set to 8 in the source code) to do. The processing business operations are executed in the business thread pool. According to the RomotingCommand business request code, the corresponding processor is found in the processorTable local cache variable and encapsulated into the task, and then submitted to the corresponding business processor processing thread pool for execution (sendMessageExecutor,). Take sending a message, for example, the "M2" above. The thread pool continues to increase in several steps from entry to business logic, which is related to the complexity of each step. The more complex the thread pool is, the wider the concurrent channel is required. From 38ece0f1558290b6bdb4b4ed9b130baab98984cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E8=89=AF=E5=9B=BD?= Date: Sun, 3 Mar 2019 15:52:23 +0800 Subject: [PATCH 082/122] [RIP-78] update Design_LoadBlancing.md in RocketMQ docs/en --- docs/en/Design_LoadBlancing.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index b93b6ea86f..9ae744dcd7 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,11 +1,11 @@ -## 4 Load Balancing -Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. +## Load Balancing +Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at constumer side when subscribing messages. -### 4.1 Producer Load Balancing +### 1 Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 4.2 Consumer Load Balancing +### 2 Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. 1， Heartbeat Packet Sending on Consumer side From aa3d6095fb919915736164480f070d837b371e74 Mon Sep 17 00:00:00 2001 From: dust6174 Date: Sun, 3 Mar 2019 16:05:53 +0800 Subject: [PATCH 083/122] [RIP-9]Delete the Introduction and underlines --- docs/en/Operations_Trace.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/Operations_Trace.md b/docs/en/Operations_Trace.md index 5ac16a6533..d6fce56de3 100644 --- a/docs/en/Operations_Trace.md +++ b/docs/en/Operations_Trace.md @@ -101,5 +101,4 @@ The initialization of `DefaultMQProducer` and `DefaultMQPushConsumer` instances DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_JODIE_1",true,"Topic_test11111"); ...... - ``` \ No newline at end of file From 084a94113e3c0abc49dc41823a604e7d91a75195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E8=89=AF=E5=9B=BD?= Date: Sun, 3 Mar 2019 16:08:58 +0800 Subject: [PATCH 084/122] [RIP-70] update Deployment.md in RocketMQ docs/en --- docs/en/Deployment.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index 5d74c163d5..beb0c3e33a 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -1,8 +1,6 @@ -# Deployment Architectures and Setup Steps +# Installation Guides -## Cluster Setup - -### 1 Single Master mode +### 1 Single Master Mode This is the simplest, but also the riskiest, mode that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build. From da32267b30e1d36cb043b58398ff93a05f2d2079 Mon Sep 17 00:00:00 2001 From: liyangbing Date: Sun, 3 Mar 2019 16:12:12 +0800 Subject: [PATCH 085/122] [RIP-9] Polish Operations_Trace.md --- docs/en/Operations_Trace.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/en/Operations_Trace.md b/docs/en/Operations_Trace.md index b6c5d45620..5b90cf923d 100644 --- a/docs/en/Operations_Trace.md +++ b/docs/en/Operations_Trace.md @@ -1,10 +1,8 @@ # Message Trace -## Introduction - This document focuses on how to quickly deploy and use RocketMQ clusters that support message trace features -## 1. Key Attributes of Message Trace Data +## 1 Key Attributes of Message Trace Data | Producer | Consumer | Broker | | ---------------- | ----------------- | ------------ | @@ -13,7 +11,7 @@ This document focuses on how to quickly deploy and use RocketMQ clusters that su | whether the message was sent successfully | Whether the message was successfully consumed | The Key of the message | | Time spent sending | Time spent consuming | Tag of the message | -## 2. Support for Message Trace Cluster Deployment +## 2 Support for Message Trace Cluster Deployment ### 2.1 Broker Configuration Fille @@ -46,7 +44,7 @@ For scenarios with large amount of trace message data , one of the Broker nodes ### 2.4 Start the Broker that Starts the MessageTrace `nohup sh mqbroker -c ../conf/2m-noslave/broker-a.properties &` -## 3. Save the Topic Definition of Message Trace +## 3 Save the Topic Definition of Message Trace RocketMQ's message trace feature supports two ways to store trace data: ### 3.1 System-level TraceTopic @@ -55,7 +53,7 @@ By default, message track data is stored in the system-level TraceTopic(names： ### 3.2 Custom TraceTopic If the user is not prepared to store the message track data in the system-level default TraceTopic, you can also define and create a user-level Topic to save the track (that is, to create a regular Topic to save the message track data)。The following section introduces how the Client interface supports the user-defined TraceTopic. -## 4. Client Practices that Support Message Trace +## 4 Client Practices that Support Message Trace In order to reduce as much as possible the transformation work of RocketMQ message trace feature used in the user service system, the author added a switch parameter (**enableMsgTrace**) to the original interface in the design to realize whether the message trace is opened or not. ### 4.1 Opening the Message Trace when Sending the Message From 09f8b9cf095c05cc87e7eda1a85dd1416880d219 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E8=89=AF=E5=9B=BD?= Date: Sun, 3 Mar 2019 16:17:53 +0800 Subject: [PATCH 086/122] Modify Deployment.md in RocketMQ docs/en --- docs/en/Configuration_System.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..4c99296bcf 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: -server -Xms8g -Xmx8g -Xmn4g @@ -24,13 +24,13 @@ As for garbage collection, G1 collector with JDK 1.8 is recommended: These GC options looks a little aggressive, but it’s proved to have good performance in our production environment -Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.So use rolling GC log file is recommended: +Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.so rolling GC log file is recommended: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m -If write GC file will increase latency of broker, consider redirect GC log file to a memory file system: +If writing a GC file increases the latency of broker, please consider redirecting it to memory file system: -Xloggc:/dev/shm/mq_gc_%p.log123 From 3c5e16b759a7b870af311d736cfb274d4e52c6a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E8=89=AF=E5=9B=BD?= Date: Sun, 3 Mar 2019 17:44:54 +0800 Subject: [PATCH 087/122] [RIP-78] update Example_Filter.md in RocketMQ docs/en --- docs/en/Configuration_System.md | 6 +++--- docs/en/Deployment.md | 6 ++++-- docs/en/Design_LoadBlancing.md | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 4c99296bcf..2f29c3bbb1 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this: -server -Xms8g -Xmx8g -Xmn4g @@ -24,13 +24,13 @@ As for garbage collection, G1 collector with JDK 1.8 is recommended: These GC options looks a little aggressive, but it’s proved to have good performance in our production environment -Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.so rolling GC log file is recommended: +Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.So use rolling GC log file is recommended: -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m -If writing a GC file increases the latency of broker, please consider redirecting it to memory file system: +If write GC file will increase latency of broker, consider redirect GC log file to a memory file system: -Xloggc:/dev/shm/mq_gc_%p.log123 diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index beb0c3e33a..5d74c163d5 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -1,6 +1,8 @@ -# Installation Guides +# Deployment Architectures and Setup Steps -### 1 Single Master Mode +## Cluster Setup + +### 1 Single Master mode This is the simplest, but also the riskiest, mode that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build. diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index 9ae744dcd7..b93b6ea86f 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,11 +1,11 @@ -## Load Balancing -Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at constumer side when subscribing messages. +## 4 Load Balancing +Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. -### 1 Producer Load Balancing +### 4.1 Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 2 Consumer Load Balancing +### 4.2 Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. 1， Heartbeat Packet Sending on Consumer side From 77c181be40df337e52369315bf28d8b9d626cced Mon Sep 17 00:00:00 2001 From: shudo1219 Date: Sun, 3 Mar 2019 18:04:56 +0800 Subject: [PATCH 088/122] [RIP-9] Update Configuration_System.md in RocketMQ docs/en:some problems in grammer --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..a129d5572a 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -36,7 +36,7 @@ If write GC file will increase latency of broker, consider redirect GC log file ## 2 Linux Kernel Parameters ## -There is a os.sh script that lists a lot of kernel parameters in folder bin which can be used for production use with minor changes. Below parameters need attention, and more details please refer to documentation for /proc/sys/vm/*. +There is an os.sh script that lists a lot of kernel parameters in folder bin which can be used for production enviroment with minor changes. Below parameters need attention, if need more details please refer to documentation for /proc/sys/vm/*. From ce65b7b3873937afd9cabe6b08bed452dee53312 Mon Sep 17 00:00:00 2001 From: MoCha <42431476+MoCha4J@users.noreply.github.com> Date: Sun, 3 Mar 2019 18:57:49 +0800 Subject: [PATCH 089/122] Update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..76de890181 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: -server -Xms8g -Xmx8g -Xmn4g From f426f4f47343165b0241c7b580d1fa38048723c4 Mon Sep 17 00:00:00 2001 From: Veryfirefly <993610942@qq.com> Date: Sun, 3 Mar 2019 19:13:19 +0800 Subject: [PATCH 090/122] format --- docs/en/CLITools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/CLITools.md b/docs/en/CLITools.md index 06b414b095..5eddc85350 100644 --- a/docs/en/CLITools.md +++ b/docs/en/CLITools.md @@ -262,7 +262,7 @@ Before introducing the mqadmin management tool, the following points need to be height:244.0pt;border-top:none;width:133pt'> clusterList View cluster information, cluster, brokerName, brokerId, TPS, and so on. + border-top:none;width:131pt'>View cluster information, cluster, brokerName, brokerId, TPS, and so on -m Print more information (add print to # InTotalYest, #OutTotalYest, #InTotalToday ,#OutTotalToday) From 540dd7faec5314aedf9e34e9e8236fbc22416e76 Mon Sep 17 00:00:00 2001 From: yemyaidr <17682303621@163.com> Date: Sun, 3 Mar 2019 20:38:37 +0800 Subject: [PATCH 091/122] Update Operations_Producer.md --- docs/en/Operations_Consumer.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/en/Operations_Consumer.md b/docs/en/Operations_Consumer.md index c5a37e5dc6..84047dfc92 100644 --- a/docs/en/Operations_Consumer.md +++ b/docs/en/Operations_Consumer.md @@ -2,24 +2,24 @@ ---- -### 2.1 Consumption process idempotent +### 1 Consumption process idempotent RocketMQ cannot avoid Exactly-Once, so if the business is very sensitive to consumption repetition, it is important to perform deduplication at the business level. Deduplication can be done with a relational database. First, you need to determine the unique key of the message, which can be either msgId or a unique identifier field in the message content, such as the order Id. Determine if a unique key exists in the relational database before consumption. If it does not exist, insert it and consume it, otherwise skip it. (The actual process should consider the atomic problem, determine whether there is an attempt to insert, if the primary key conflicts, the insertion fails, skip directly) -### 2.2 Slow message processing +### 2 Slow message processing -#### 2.2.1 Increase consumption parallelism +#### 2.1 Increase consumption parallelism Most messages consumption behaviors are IO-intensive, That is, it may be to operate the database, or call RPC. The consumption speed of such consumption behavior lies in the throughput of the back-end database or the external system. By increasing the consumption parallelism, the total consumption throughput can be increased, but the degree of parallelism is increased to a certain extent. Instead it will fall.Therefore, the application must set a reasonable degree of parallelism. There are several ways to modify the degree of parallelism of consumption as follows: * Under the same ConsumerGroup, increase the degree of parallelism by increasing the number of Consumer instances (note that the Consumer instance that exceeds the number of subscription queues is invalid). Can be done by adding machines, or by starting multiple processes on an existing machine. * Improve the consumption parallel thread of a single Consumer by modifying the parameters consumeThreadMin and consumeThreadMax. -#### 2.2.2 Batch mode consumption +#### 2.2 Batch mode consumption Some business processes can increase consumption throughput to a large extent if they support batch mode consumption. For example, order deduction application, it takes 1s to process one order at a time, and it takes only 2s to process 10 orders at a time. In this way, the throughput of consumption can be greatly improved. By setting the consumer's consumeMessageBatchMaxSize to return a parameter, the default is 1, that is, only one message is consumed at a time, for example, set to N, then the number of messages consumed each time is less than or equal to N. -#### 2.2.3 Skip non-critical messages +#### 2.3 Skip non-critical messages When a message is accumulated, if the consumption speed cannot keep up with the transmission speed, if the service does not require high data, you can choose to discard the unimportant message. For example, when the number of messages in a queue is more than 100,000 , try to discard some or all of the messages, so that you can quickly catch up with the speed of sending messages. The sample code is as follows: @@ -40,7 +40,7 @@ public ConsumeConcurrentlyStatus consumeMessage( } ``` -#### 2.2.4 Optimize each message consumption process +#### 2.4 Optimize each message consumption process For example, the consumption process of a message is as follows: @@ -52,7 +52,7 @@ For example, the consumption process of a message is as follows: There are 4 interactions with the DB in the consumption process of this message. If it is calculated by 5ms each time, it takes a total of 20ms. If the business calculation takes 5ms, then the total time is 25ms, So if you can optimize 4 DB interactions to 2 times, the total time can be optimized to 15ms, which means the overall performance is increased by 40%. Therefore, if the application is sensitive to delay, the DB can be deployed on the SSD hard disk. Compared with the SCSI disk, the former RT will be much smaller. -### 2.3 Print Log +### 3 Print Log If the amount of messages is small, it is recommended to print the message in the consumption entry method, consume time, etc., to facilitate subsequent troubleshooting. @@ -68,33 +68,33 @@ public ConsumeConcurrentlyStatus consumeMessage( If you can print the time spent on each message, it will be more convenient when troubleshooting online problems such as slow consumption. -### 2.4 Other consumption suggestions +### 4 Other consumption suggestions -#### 2.4.1、Consumer Group and Subscriptions +#### 4.1、Consumer Group and Subscriptions The first thing you should be aware of is that different Consumer Group can consume the same topic independently, and each of them will have their own consuming offsets. Please make sure each Consumer within the same Group to subscribe the same topics. -#### 2.4.2、Orderly +#### 4.2、Orderly The Consumer will lock each MessageQueue to make sure it is consumed one by one in order. This will cause a performance loss, but it is useful when you care about the order of the messages. It is not recommended to throw exceptions, you can return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT instead. -#### 2.4.3、Concurrently +#### 4.3、Concurrently As the name tells, the Consumer will consume the messages concurrently. It is recommended to use this for good performance. It is not recommended to throw exceptions, you can return ConsumeConcurrentlyStatus.RECONSUME_LATER instead. -#### 2.4.4、Consume Status +#### 4.4、Consume Status For MessageListenerConcurrently, you can return RECONSUME_LATER to tell the consumer that you can not consume it right now and want to reconsume it later. Then you can continue to consume other messages. For MessageListenerOrderly, because you care about the order, you can not jump over the message, but you can return SUSPEND_CURRENT_QUEUE_A_MOMENT to tell the consumer to wait for a moment. -#### 2.4.5、Blocking +#### 4.5、Blocking It is not recommend to block the Listener, because it will block the thread pool, and eventually may stop the consuming process. -#### 2.4.6、Thread Number +#### 4.6、Thread Number The consumer use a ThreadPoolExecutor to process consuming internally, so you can change it by setting setConsumeThreadMin or setConsumeThreadMax. -#### 2.4.7、ConsumeFromWhere +#### 4.7、ConsumeFromWhere When a new Consumer Group is established, it will need to decide whether it needs to consume the historical messages which had already existed in the Broker. CONSUME_FROM_LAST_OFFSET will ignore the historical messages, and consume anything produced after that. CONSUME_FROM_FIRST_OFFSET will consume every message existed in the Broker. You can also use CONSUME_FROM_TIMESTAMP to consume messages produced after the specified timestamp. From 3dbe55fd8200f0a1ae4533fd3b68bee75ebeece1 Mon Sep 17 00:00:00 2001 From: zy <602128569@qq.com> Date: Sun, 3 Mar 2019 21:57:27 +0800 Subject: [PATCH 092/122] Update Configuration_System.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JVM Options :A simple JVM configurations looks like this 改为A simple JVM configuration is as follows --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..76de890181 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: -server -Xms8g -Xmx8g -Xmn4g From ad0ceb74759afde803ef24f4ff235f4f38e6cb97 Mon Sep 17 00:00:00 2001 From: ouxu Date: Sun, 3 Mar 2019 22:18:21 +0800 Subject: [PATCH 093/122] [RIP-65] Delete chinese period in CLITools.md file --- docs/en/CLITools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/CLITools.md b/docs/en/CLITools.md index 06b414b095..ff0ca2c54f 100644 --- a/docs/en/CLITools.md +++ b/docs/en/CLITools.md @@ -283,7 +283,7 @@ Before introducing the mqadmin management tool, the following points need to be clusterRT Send message to detect each broker RT of the cluster.the message send to ${BrokerName} Topic。 + border-top:none;width:131pt'>Send message to detect each broker RT of the cluster.the message send to ${BrokerName} Topic -a amount，total number per probe，RT = Total time/amount From 18580133e037c783edee9e859bebc2da3290d6ce Mon Sep 17 00:00:00 2001 From: May <986987740@qq.com> Date: Sun, 3 Mar 2019 22:33:05 +0800 Subject: [PATCH 094/122] Change Configuration_System : We recommend set 655350 - We recommend setting 655350 --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..f5e629bd8c 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -59,7 +59,7 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic -- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend set 655350 for file descriptors. +- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend setting 655350 for file descriptors. From 62acf7cd16643a869da9f1b16d0356133ab4efe6 Mon Sep 17 00:00:00 2001 From: shudo1219 Date: Sun, 3 Mar 2019 22:54:50 +0800 Subject: [PATCH 095/122] [RIP-9] update Concept.md in RocketMQ docs/en --- docs/en/Concept.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Concept.md b/docs/en/Concept.md index e1e5ecbb42..490b98c156 100644 --- a/docs/en/Concept.md +++ b/docs/en/Concept.md @@ -6,7 +6,7 @@ RocketMQ message model is mainly composed of Producer, Broker and Consumer. The The broker is an independent server during actual deployment, and each broker can store messages from multiple topics. Even messages from the same topic can be stored in the different brokers by sharding strategy. The message queue is used to store physical offsets of messages, and the message addresses are stored in seperate queues. The consumer group consists of multiple consumer instances. ## 2 Producer -The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the systems to brokers. RocketMQ provides multiple paradigms of sending: synchronous, asynchronous, sequential and one-way. Both synchronous and asynchronous methods require the confirmation information return from the Broker, but one-way don't need. +The Producer is responsible for producing messages, typically by business systems. It sends messages generated by the systems to brokers. RocketMQ provides multiple paradigms of sending: synchronous, asynchronous, sequential and one-way. Both synchronous and asynchronous methods require the confirmation information return from the Broker, but one-way method does not require it. ## 3 Consumer The Consumer is responsible for consuming messages, typically the background system is responsible for asynchronous consumption. The consumer pulls messages from brokers and feeds them into application. From the perspective of user, two types of consumers are provided: pull consumer and push consumer. ## 4 Topic @@ -16,7 +16,7 @@ As the role of the transfer station, the Broker Server stores and forwards messa ## 6 Name Server The Name Server serves as the provider of routing service. The producer or the consumer can find the list of broker IP addresses for each topic through name server. Multiple name servers can be deployed in one cluster, but they are independent of each other and do not exchange information. ## 7 Pull Consumer -A type of Consumer, the application pulls messages from brokers by actively invoking the consumer's pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process. +A type of Consumer, the application pulls messages from brokers by actively invoking the consumer pull message method, and the application has the advantages of controlling the timing and frequency of pulling messages. Once the batch of messages is pulled, user application will initiate consuming process. ## 8 Push Consumer A type of Consumer. Under this high real-time performance mode, it will push the message to the consumer actively when the Broker receives the data. ## 9 Producer Group From 3028bda2da3d3385f6d107563adb4092a552b793 Mon Sep 17 00:00:00 2001 From: hackjeremy <350500200@qq.com> Date: Sun, 3 Mar 2019 23:02:43 +0800 Subject: [PATCH 096/122] Example_Delay.md [RIP-77]Modify the last paragraph and rid of punctuation between each section number and title --- docs/en/Example_Delay.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/Example_Delay.md b/docs/en/Example_Delay.md index 80785256d9..fe41aed92b 100644 --- a/docs/en/Example_Delay.md +++ b/docs/en/Example_Delay.md @@ -1,6 +1,6 @@ # Schedule example -### 1.Start consumer to wait for incoming subscribed messages +### 1 Start consumer to wait for incoming subscribed messages ```java import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; @@ -35,7 +35,7 @@ public class ScheduledMessageConsumer { } ``` -### 2.Send scheduled messages +### 2 Send scheduled messages ```java import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -64,15 +64,15 @@ public class ScheduledMessageProducer { } ``` -### 3.Verification +### 3 Verification You should see messages are consumed about 10 seconds later than their storing time. -### 4.Use scenarios for scheduled messages +### 4 Use scenarios for scheduled messages For example, in e-commerce, if an order is submitted, a delay message can be sent, and the status of the order can be checked after 1 hour. If the order is still unpaid, the order can be cancelled and the inventory released. -### 5.Restrictions on the use of scheduled messages +### 5 Restrictions on the use of scheduled messages ```java // org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -82,4 +82,4 @@ private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m Nowadays RocketMq does not support any time delay. It needs to set several fixed delay levels, which correspond to level 1 to 18 from 1s to 2h. Message consumption failure will enter the delay message queue. Message sending time is related to the set delay level and the number of retries. - See `SendMessageProcessor.java` \ No newline at end of file + See `SendMessageProcessor.java` From c37ccf09cdd4ce06654f67434490257a061458ab Mon Sep 17 00:00:00 2001 From: May <986987740@qq.com> Date: Sun, 3 Mar 2019 23:08:38 +0800 Subject: [PATCH 097/122] New Change Configuration_System : We recommend set 655350 - We recommend setting 655350 --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index f5e629bd8c..8a21032ebb 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -59,7 +59,7 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic -- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend setting 655350 for file descriptors. +- **File descriptor limits**, RocketMQ needs open file descriptors for files(CommitLog and ConsumeQueue) and network connections. We recommend setting 655350 for file descriptors. From 4abcb0b0f05b8b4dffaf82f4c1396a7843a64181 Mon Sep 17 00:00:00 2001 From: zhanguohuang Date: Sun, 3 Mar 2019 23:27:56 +0800 Subject: [PATCH 098/122] Number the titles of Configuration_Client.md --- docs/en/Configuration_Client.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/en/Configuration_Client.md b/docs/en/Configuration_Client.md index b4a14a1d65..4b6d2fe39a 100644 --- a/docs/en/Configuration_Client.md +++ b/docs/en/Configuration_Client.md @@ -2,7 +2,7 @@ Relative to RocketMQ's Broker cluster, producers and consumers are client. In this section, it mainly describes the common behavior configuration of producers and consumers. ​ -### Client Addressing mode +### 1 Client Addressing mode ```RocketMQ``` can let client find the ```Name Server```, and then find the ```Broker```by the ```Name Server```. Followings show a variety of configurations, and priority level from highly to lower, the highly priority configurations can override the lower priority configurations. @@ -36,11 +36,11 @@ By default, the client accesses the HTTP server every 2 minutes, and update the ``` HTTP static server addressing is recommended, because it is simple client deployment, and the Name Server cluster can be upgraded hot. -### Client Configuration +### 2 Client Configuration ```DefaultMQProducer```,```TransactionMQProducer```,```DefaultMQPushConsumer```,```DefaultMQPullConsumer``` all extends the ```ClientConfig``` Class, ```ClientConfig``` as the client common configuration class. Client configuration style like getXXX,setXXX, each of the parameters can config by spring and also config their in the code. Such as the ```namesrvAddr``` parameter: ```producer.setNamesrvAddr("192.168.0.1:9876")```, same with the other parameters. -#### Client Common Configuration +#### 2.1 Client Common Configuration | Pamater Name | Default Value | Description | | ----------------------------- | ------- | ------------------------------------------------------------ | @@ -52,7 +52,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | heartbeatBrokerInterval | 30000 | The heartbeat interval, in milliseconds, is sent to the Broker | | persistConsumerOffsetInterval | 5000 | The persistent Consumer consumes the progress interval in milliseconds | -#### Producer Configuration +#### 2.2 Producer Configuration | Pamater Name | Default Value | Description | | -------------------------------- | ---------------- | ------------------------------------------------------------ | @@ -70,7 +70,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | checkRequestHoldMax | 2000 | Producer local buffer request queue size when Broker look back Producer transaction status | | RPCHook | null | This parameter is passed in when the Producer is creating, including the pre-processing before the message sending and the processing after the message response. The user can do some security control or other operations in the first interface.| -#### PushConsumer Configuration +#### 2.3 PushConsumer Configuration | Pamater Name | Default Value | Description | | ---------------------------- | ----------------------------- | ------------------------------------------------------------ | @@ -91,7 +91,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | consumeMessageBatchMaxSize | 1 | Batch consume message | | pullBatchSize | 32 | Batch pull message | -#### PullConsumer Configuration +#### 2.4 PullConsumer Configuration | Pamater Name | Default Value | Description | | -------------------------------- | ----------------------------- | ------------------------------------------------------------ | @@ -105,7 +105,7 @@ HTTP static server addressing is recommended, because it is simple client deploy | registerTopics | | Collection of registered topics | | allocateMessageQueueStrategy | AllocateMessageQueueAveragely | Implements strategy about Rebalance algorithm | -#### Message Data Structure +#### 2.5 Message Data Structure | Field Name | Default Value | Description | | -------------- | ------ | ------------------------------------------------------------ | From 4d9fd188ced70b3cfb3a152f35387588d4b1e524 Mon Sep 17 00:00:00 2001 From: hfb Date: Sun, 3 Mar 2019 23:38:15 +0800 Subject: [PATCH 099/122] update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..7871fccfdd 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -55,7 +55,7 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic -- **vm.swappiness**, define how aggressive the kernel will swap memory pages. Higher values will increase agressiveness, lower values decrease the amount of swap. 10 is recommended for this value to avoid swap latency. +- **vm.swappiness**, defines how aggressive the kernel will swap memory pages. Higher values will increase agressiveness, lower values decrease the amount of swap. 10 is recommended for this value to avoid swap latency. From 05c5fabd24ee5c910061dffd5d6813723bd5fe10 Mon Sep 17 00:00:00 2001 From: pengbin <723454754@qq.com> Date: Mon, 4 Mar 2019 08:51:30 +0800 Subject: [PATCH 100/122] [RIP-9] Update Deployment.md in RocketMQ --- docs/en/Deployment.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index 5d74c163d5..ec12f7ff7e 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -69,7 +69,7 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-noslave/broker ... ``` -The boot command shown above is used in the case of a single NameServer.For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.161.2: 9876. +The boot command shown above is used in the case of a single NameServer. For clusters of multiple NameServer, the address list after the -n argument in the broker boot command is separated by semicolons, for example, 192.168.1.1: 9876;192.161.2: 9876. ### 3 Multiple Master And Multiple Slave Mode-Asynchronous replication @@ -156,4 +156,4 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties & ``` -The above Master and Slave are paired by specifying the same config named "brokerName", the "brokerId" of the master node must be 0, and the "brokerId" of the slave node must be greater than 0. \ No newline at end of file +The above Master and Slave are paired by specifying the same config named "brokerName", the "brokerId" of the master node must be 0, and the "brokerId" of the slave node must be greater than 0. From 5dd62e312fc0c99c19bc86ce47b512a0e4d04670 Mon Sep 17 00:00:00 2001 From: pengbin <723454754@qq.com> Date: Mon, 4 Mar 2019 09:06:23 +0800 Subject: [PATCH 101/122] [RIP-9] Changed'.For clusters of multiple NameServer' . For clusters of multiple NameServer --- docs/en/Deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index ec12f7ff7e..36aad076bf 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -156,4 +156,4 @@ $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker $ nohup sh mqbroker -n 192.168.1.1:9876 -c $ROCKETMQ_HOME/conf/2m-2s-sync/broker-b-s.properties & ``` -The above Master and Slave are paired by specifying the same config named "brokerName", the "brokerId" of the master node must be 0, and the "brokerId" of the slave node must be greater than 0. +The above Master and Slave are paired by specifying the same config named "brokerName", the "brokerId" of the master node must be 0, and the "brokerId" of the slave node must be greater than 0. \ No newline at end of file From eda0e64f65abea55bb933e6ff07bd6c87fea24f0 Mon Sep 17 00:00:00 2001 From: zhoushuyi Date: Mon, 4 Mar 2019 10:45:44 +0800 Subject: [PATCH 102/122] [RIP-9] update Deployment.md in docs/en --- docs/en/Deployment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Deployment.md b/docs/en/Deployment.md index 5d74c163d5..cc4ea564b7 100644 --- a/docs/en/Deployment.md +++ b/docs/en/Deployment.md @@ -4,7 +4,7 @@ ### 1 Single Master mode -This is the simplest, but also the riskiest, mode that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build. +This is the simplest but also the riskiest mode, that makes the entire service unavailable once the broker restarts or goes down. Production environments are not recommended, but can be used for local testing and development. Here are the steps to build. **1）Start NameServer** From ccb934b157dc46b7fb94eb68332d71485c45dd4c Mon Sep 17 00:00:00 2001 From: jiangkun <525278372@qqcom> Date: Mon, 4 Mar 2019 14:10:50 +0800 Subject: [PATCH 103/122] Update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..0426a4e1a2 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -30,7 +30,7 @@ Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m -If write GC file will increase latency of broker, consider redirect GC log file to a memory file system: +If writing a GC file increases the latency of broker, please consider redirecting it to memory file system: -Xloggc:/dev/shm/mq_gc_%p.log123 From 12ccd50806bafae2d8c4a6fe0cff668a96ecb8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=9E=83=E8=9F=B9=E4=BB=A5=E6=AD=BB?= Date: Tue, 5 Mar 2019 17:55:50 +0800 Subject: [PATCH 104/122] [RIP-9] Add the system configuration option doc: Direct ByteBuffer memory size [RIP-9] Add the system configuration option doc: Direct ByteBuffer memory size --- docs/en/Configuration_System.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 2f29c3bbb1..6354ffe827 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -8,6 +8,10 @@ The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx -server -Xms8g -Xmx8g -Xmn4g +Direct ByteBuffer memory size setting. Full GC will be triggered when the Direct ByteBuffer up to the specified size: + + -XX:MaxDirectMemorySize=15g + If you don’t care about the boot time of RocketMQ broker, pre-touch the Java heap to make sure that every page will be allocated during JVM initialization is a better choice. Those who don’t care about the boot time can enable it: -XX:+AlwaysPreTouch From 8b8051fce96b030214cab8e6e893d9e524733dd1 Mon Sep 17 00:00:00 2001 From: liruibo Date: Tue, 5 Mar 2019 19:29:05 +0800 Subject: [PATCH 105/122] fix Design_Remoting.md issues no 103-105 --- docs/en/Design_Remoting.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/Design_Remoting.md b/docs/en/Design_Remoting.md index ff2c99d2e6..39690d195b 100644 --- a/docs/en/Design_Remoting.md +++ b/docs/en/Design_Remoting.md @@ -2,10 +2,10 @@ RocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows: (1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds. (2) When message producer Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. -(3) Message producer Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message. -(4) After message consumer Consumer get the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them. +(3) Message Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message. +(4) After message Consumer gets the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them. -From 1) ~ 3 above, we can see that both Producer, Broker and NameServer communicate with each other(only part of MQ communication is mentioned here), so how to design a good network communication module is very important in MQ. It will determine the overall messaging capability and final performance of the RocketMQ cluster. +From 1) ~ 3) above, we can see that both Producer, Broker and NameServer communicate with each other(only part of MQ communication is mentioned here), so how to design a good network communication module is very important in MQ. It will determine the overall messaging capability and final performance of the RocketMQ cluster. rocketmq-remoting module is the module responsible for network communication in RocketMQ message queue. It is relied on and referenced by almost all other modules (such as rocketmq-client,rocketmq-broker,rocketmq-namesrv) that need network communication. In order to realize the efficient data request and reception between the client and the server, the RocketMQ message queue defines the communication protocol and extends the communication module on the basis of Netty. From 9633a1f067aa73bcb5ec6dec79fc7f23aa9bbecd Mon Sep 17 00:00:00 2001 From: yazong <403539228@qq.com> Date: Tue, 5 Mar 2019 20:06:47 +0800 Subject: [PATCH 106/122] modify (2) --- docs/en/Design_Remoting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Design_Remoting.md b/docs/en/Design_Remoting.md index ff2c99d2e6..b90413ec9a 100644 --- a/docs/en/Design_Remoting.md +++ b/docs/en/Design_Remoting.md @@ -1,7 +1,7 @@ ## 2 Communication Mechanism RocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows: (1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds. -(2) When message producer Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. +(2) When message Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. (3) Message producer Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message. (4) After message consumer Consumer get the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them. From 1bc1f22ce3e6b7621abab5e9946ca84281d20aa5 Mon Sep 17 00:00:00 2001 From: dongeforever Date: Tue, 5 Mar 2019 20:46:14 +0800 Subject: [PATCH 107/122] Add copywright for some files --- distribution/bin/dledger/fast-try.sh | 17 ++++++++++++++++- pom.xml | 1 + .../apache/rocketmq/store/StoreTestBase.java | 16 ++++++++++++++++ .../store/dledger/DLedgerCommitlogTest.java | 16 ++++++++++++++++ .../store/dledger/MessageStoreTestBase.java | 16 ++++++++++++++++ .../store/dledger/MixCommitlogTest.java | 16 ++++++++++++++++ .../dledger/DLedgerProduceAndConsumeIT.java | 16 ++++++++++++++++ 7 files changed, 97 insertions(+), 1 deletion(-) diff --git a/distribution/bin/dledger/fast-try.sh b/distribution/bin/dledger/fast-try.sh index e536d455fa..ff8a96036d 100644 --- a/distribution/bin/dledger/fast-try.sh +++ b/distribution/bin/dledger/fast-try.sh @@ -1,4 +1,19 @@ -#! /bin/bash +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. ## Revise the base dir CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)" diff --git a/pom.xml b/pom.xml index 83b564a01a..7a349369c0 100644 --- a/pom.xml +++ b/pom.xml @@ -259,6 +259,7 @@ */target/** */*.iml docs/** + localbin/** diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java index aa2919d821..0dcb3b254c 100644 --- a/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestBase.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.store; import java.io.File; diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java index a6acbdb8d9..d0829d122a 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/DLedgerCommitlogTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.DLedgerServer; diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java index 78a6837727..e45178aaee 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MessageStoreTestBase.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.store.dledger; import io.openmessaging.storage.dledger.DLedgerConfig; diff --git a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java index 540486d2aa..a5242a3510 100644 --- a/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/dledger/MixCommitlogTest.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.store.dledger; import java.util.UUID; diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java index 7fd229749f..e2d9db1983 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.test.base.dledger; import java.util.UUID; From 275882577b965e715e4371f7af50489440a08e39 Mon Sep 17 00:00:00 2001 From: Yalong Li Date: Thu, 7 Mar 2019 15:50:13 +0800 Subject: [PATCH 108/122] [RIP-9]Modify Design_Remoting.md --- docs/en/Design_Remoting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Design_Remoting.md b/docs/en/Design_Remoting.md index ff2c99d2e6..b90413ec9a 100644 --- a/docs/en/Design_Remoting.md +++ b/docs/en/Design_Remoting.md @@ -1,7 +1,7 @@ ## 2 Communication Mechanism RocketMQ message queue cluster mainly includes four roles: NameServer, Broker (Master/Slave), Producer and Consumer. The basic communication process is as follows: (1) After Broker start-up, it needs to complete one operation: register itself to NameServer, and then report Topic routing information to NameServer at regular intervals of 30 seconds. -(2) When message producer Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. +(2) When message Producer sends a message as a client, it needs to obtain routing information from the local cache TopicPublishInfoTable according to the Topic of the message. If not, it will be retrieved from NameServer and update to local cache, at the same time, Producer will retrieve routing information from NameServer every 30 seconds by default. (3) Message producer Producer chooses a queue to send the message according to the routing information obtained in 2); Broker receives the message and records it in disk as the receiver of the message. (4) After message consumer Consumer get the routing information according to 2) and complete the load balancing of the client, then select one or several message queues to pull messages and consume them. From 8497429ededd38fbb433ebcd26b575492650ce34 Mon Sep 17 00:00:00 2001 From: Suo Caiyuan Date: Thu, 7 Mar 2019 16:01:40 +0800 Subject: [PATCH 109/122] [RIP-9] Update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 0426a4e1a2..1ccd6927a3 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -51,7 +51,7 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic -- **vm.max_map_count**, limits the maximum number of memory map areas a process may have. RocketMQ will use mmap to load CommitLog and ConsumeQueue, so set a bigger value for this parameter is recommended. +- **vm.max_map_count**, limits the maximum number of memory map areas a process may have. RocketMQ will use mmap to load CommitLog and ConsumeQueue, so a bigger value is recommended. From 06eeaed151d1e263d60ce59a17f7ac4d06de0b32 Mon Sep 17 00:00:00 2001 From: hdchen <14106312@qq.com> Date: Thu, 7 Mar 2019 19:11:54 +0800 Subject: [PATCH 110/122] [RIP-10]ConsumeMessageConcurrentlyServiceTest maybe effect other case,Ignore part test function --- .../impl/consumer/ConsumeMessageConcurrentlyServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java index 15a261f430..921743c6be 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyServiceTest.java @@ -36,6 +36,7 @@ import org.apache.rocketmq.common.stats.StatsItemSet; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -140,6 +141,7 @@ public class ConsumeMessageConcurrentlyServiceTest { pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); } + @Ignore @Test public void testPullMessage_ConsumeSuccess() throws InterruptedException, RemotingException, MQBrokerException, NoSuchFieldException,Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); From 5fc047d5080a431451beb5c21a491b66ef526234 Mon Sep 17 00:00:00 2001 From: zy <602128569@qq.com> Date: Thu, 7 Mar 2019 20:06:28 +0800 Subject: [PATCH 111/122] Update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 76de890181..82fcab1600 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows -server -Xms8g -Xmx8g -Xmn4g From 6f17c8e8d34624fd54fdebb38b49e2169d49f795 Mon Sep 17 00:00:00 2001 From: zy <602128569@qq.com> Date: Thu, 7 Mar 2019 20:07:08 +0800 Subject: [PATCH 112/122] Update Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 82fcab1600..76de890181 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -4,7 +4,7 @@ This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: -server -Xms8g -Xmx8g -Xmn4g From 3043b05de07e800b6355e441476e13a3681cb8ab Mon Sep 17 00:00:00 2001 From: fengjian Date: Thu, 7 Mar 2019 20:07:23 +0800 Subject: [PATCH 113/122] Modify English document for Configuration_System.md --- docs/en/Configuration_System.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 941dfa70d0..07b4b28a57 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -64,4 +64,4 @@ There is a os.sh script that lists a lot of kernel parameters in folder bin whic - **Disk scheduler**, the deadline I/O scheduler is recommended for RocketMQ, which attempts to provide a guaranteed latency for requests. - + From 1ad3de87bb2df2f970bbfee551f3f42594aaa020 Mon Sep 17 00:00:00 2001 From: dust6174 Date: Thu, 7 Mar 2019 20:41:43 +0800 Subject: [PATCH 114/122] [RIP-9]Delete the Introduction and underlines --- docs/en/Operations_Trace.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/Operations_Trace.md b/docs/en/Operations_Trace.md index d6fce56de3..6a7ea8c716 100644 --- a/docs/en/Operations_Trace.md +++ b/docs/en/Operations_Trace.md @@ -1,5 +1,7 @@ # Message Trace + + ## 1. Key Attributes of Message Trace Data | Producer | Consumer | Broker | From 3bf799c637a40e8884f28a6510cdf95ba718e35d Mon Sep 17 00:00:00 2001 From: wubiqing <48171788+wubiqing@users.noreply.github.com> Date: Thu, 7 Mar 2019 21:48:41 +0800 Subject: [PATCH 115/122] [RIP-9] update FAQ.md in RocketMQ docs/en --- docs/en/FAQ.md | 59 +++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/docs/en/FAQ.md b/docs/en/FAQ.md index d56c38e98a..a08315cb15 100644 --- a/docs/en/FAQ.md +++ b/docs/en/FAQ.md @@ -2,34 +2,39 @@ The following questions are frequently asked with regard to the RocketMQ project in general. -## General +## 1 General -### 1. Why did we create rocketmq project instead of selecting other products? +1. Why did we create rocketmq project instead of selecting other products? -Please refer to [Why RocketMQ](http://rocketmq.apache.org/docs/motivation) + Please refer to [Why RocketMQ](http://rocketmq.apache.org/docs/motivation) -### 2. Do I have to install other softeware, such as zookeeper, to use RocketMQ? +2. Do I have to install other softeware, such as zookeeper, to use RocketMQ? -No. RocketMQ can run independently. + No. RocketMQ can run independently. -## Usage +## 2 Usage ### 1. Where does the newly created Consumer ID start consuming messages? -1. If the topic sends a message within three days, then the consumer start consuming messages from the first message saved in the server. -2. If the topic sends a message three days ago, the consumer starts to consume messages from the latest message in the server, in other words, starting from the tail of message queue. -3. If such consumer is rebooted, then it starts to consume messages from the last consumption location. + 1) If the topic sends a message within three days, then the consumer start consuming messages from the first message saved in the server. + + 2) If the topic sends a message three days ago, the consumer starts to consume messages from the latest message in the server, in other words, starting from the tail of message queue. + + 3) If such consumer is rebooted, then it starts to consume messages from the last consumption location. ### 2. How to reconsume message when consumption fails? -1. Cluster consumption pattern, The consumer business logic code returns Action.ReconsumerLater, NULL, or throws an exception, if a message failed to be consumed, it will retry for up to 16 times, after that, the message would be descarded. -2. Broadcast consumption patternThe broadcaset consumption still ensures that a message is consumered at least once, but no resend option is provided. + 1) Cluster consumption pattern, The consumer business logic code returns Action.ReconsumerLater, NULL, or throws an exception, if a message failed to be consumed, it will retry for up to 16 times, after that, the message would be descarded. + + 2) Broadcast consumption patternThe broadcaset consumption still ensures that a message is consumered at least once, but no resend option is provided. ### 3. How to query the failed message if there is a consumption failure? -1. Using topic query by time, you can query messages within a period of time. -2. Using Topic and Message Id to accurately query the message. -3. Using Topic and Message Key accurately query a class of messages with the same Message Key. + 1) Using topic query by time, you can query messages within a period of time. + + 2) Using Topic and Message Id to accurately query the message. + + 3) Using Topic and Message Key accurately query a class of messages with the same Message Key. ### 4. Are messages delivered exactly once? @@ -37,10 +42,11 @@ RocketMQ ensures that all messages are delivered at least once. In most cases, t ### 5. How to add a new broker? -1. Start up a new broker and register it to the same list of name servers. -2. By default, only internal system topics and consumer groups are created automatically. If you would like to have your business topic and consumer groups on the new node, please replicate them from the existing broker. Admin tool and command lines are provided to handle this. + 1) Start up a new broker and register it to the same list of name servers. + + 2) By default, only internal system topics and consumer groups are created automatically. If you would like to have your business topic and consumer groups on the new node, please replicate them from the existing broker. Admin tool and command lines are provided to handle this. -## Configuration related +## 3 Configuration related The following answers are all default values and can be modified by configuration. @@ -60,7 +66,7 @@ consumer.setConsumeThreadMin(20); consumer.setConsumeThreadMax(20); ``` -## Errors +## 4 Errors ### 1. If you start a producer or consumer failed and the error message is producer group or consumer repeat. @@ -76,15 +82,15 @@ Solution: Fastjson version has to be upgraded to rocketmq client dependent versi ### 3. What is the impact of a broker crash. -1. Master crashes + 1) Master crashes Messages can no longer be sent to this broker set, but if you have another broker set available, messages can still be sent given the topic is present. Messages can still be consumed from slaves. -2. Some slave crash + 2) Some slave crash As long as there is another working slave, there will be no impact on sending messages. There will also be no impact on consuming messages except when the consumer group is set to consume from this slave preferably. By default, comsumer group consumes from master. -3. All slaves crash + 3) All slaves crash There will be no impact on sending messages to master, but, if the master is SYNC_MASTER, producer will get a SLAVE_NOT_AVAILABLE indicating that the message is not sent to any slaves. There will also be no impact on consuming messages except that if the consumer group is set to consume from slave preferably. By default, comsumer group consumes from master. @@ -92,9 +98,12 @@ There will be no impact on sending messages to master, but, if the master is SYN This happens when you are trying to send messages to a topic whose routing info is not available to the producer. -1. Make sure that the producer can connect to a name server and is capable of fetching routing meta info from it. -2. Make sure that name servers do contain routing meta info of the topic. You may query the routing meta info from name server through topicRoute using admin tools or web console. -3. Make sure that your brokers are sending heartbeats to the same list of name servers your producer is connecting to. -4. Make sure that the topic’s permssion is 6(rw-), or at least 2(-w-). + 1) Make sure that the producer can connect to a name server and is capable of fetching routing meta info from it. + + 2) Make sure that name servers do contain routing meta info of the topic. You may query the routing meta info from name server through topicRoute using admin tools or web console. + + 3) Make sure that your brokers are sending heartbeats to the same list of name servers your producer is connecting to. + + 4) Make sure that the topic’s permssion is 6(rw-), or at least 2(-w-). If you can’t find this topic, create it on a broker via admin tools command updateTopic or web console. From d9f189c92138d6332c2ebb8df27677c90db7319e Mon Sep 17 00:00:00 2001 From: xiaoyukid Date: Fri, 8 Mar 2019 10:36:22 +0800 Subject: [PATCH 116/122] [RIP-9]remove "**" from headline and adjust some content formate --- docs/en/Configuration_System.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/en/Configuration_System.md b/docs/en/Configuration_System.md index 261b7e5869..606007171d 100644 --- a/docs/en/Configuration_System.md +++ b/docs/en/Configuration_System.md @@ -1,10 +1,10 @@ -# The system configuration # +# The system configuration This section focuses on the configuration of the system (JVM/OS) ## **1 JVM Options** ## -The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configuration is as follows: +The latest released version of JDK 1.8 is recommended. Set the same Xms and Xmx value to prevent the JVM from resizing the heap for better performance. A simple JVM configurations looks like this: -server -Xms8g -Xmx8g -Xmn4g @@ -22,7 +22,7 @@ As for garbage collection, G1 collector with JDK 1.8 is recommended: -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -These GC options looks a little aggressive, but it’s proved to have good performance in our production environment. +These GC options looks a little aggressive, but it’s proved to have good performance in our production environment Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a small young generation to achieve this goal which will cause very frequent minor GC.So use rolling GC log file is recommended: @@ -30,13 +30,13 @@ Don’t set a too small value for -XX:MaxGCPauseMillis, otherwise JVM will use a -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m -If writing a GC file increases the latency of broker, please consider redirecting it to memory file system: +If write GC file will increase latency of broker, consider redirect GC log file to a memory file system: -Xloggc:/dev/shm/mq_gc_%p.log123 ## 2 Linux Kernel Parameters ## -There is an os.sh script that lists a lot of kernel parameters in folder bin which can be used for production enviroment with minor changes. Below parameters need attention, if need more details please refer to documentation for /proc/sys/vm/*. +There is a os.sh script that lists a lot of kernel parameters in folder bin which can be used for production use with minor changes. Below parameters need attention, and more details please refer to documentation for /proc/sys/vm/*. @@ -51,11 +51,11 @@ There is an os.sh script that lists a lot of kernel parameters in folder bin whi -- **vm.max_map_count**, limits the maximum number of memory map areas a process may have. RocketMQ will use mmap to load CommitLog and ConsumeQueue, so a bigger value is recommended. +- **vm.max_map_count**, limits the maximum number of memory map areas a process may have. RocketMQ will use mmap to load CommitLog and ConsumeQueue, so set a bigger value for this parameter is recommended. -- **vm.swappiness**, defines how aggressive the kernel will swap memory pages. Higher values will increase agressiveness, lower values decrease the amount of swap. 10 is recommended for this value to avoid swap latency. +- **vm.swappiness**, define how aggressive the kernel will swap memory pages. Higher values will increase agressiveness, lower values decrease the amount of swap. 10 is recommended for this value to avoid swap latency. @@ -64,4 +64,4 @@ There is an os.sh script that lists a lot of kernel parameters in folder bin whi - **Disk scheduler**, the deadline I/O scheduler is recommended for RocketMQ, which attempts to provide a guaranteed latency for requests. - + From 02e2abb5eecf76ea2cd5ab2d30a6927441c60067 Mon Sep 17 00:00:00 2001 From: zhangjidi2016 <1017543663@qq.com> Date: Fri, 8 Mar 2019 17:17:51 +0800 Subject: [PATCH 117/122] [RIP-10] Add test cases for ConsumerRunningInfo (#923) --- .../body/ConsumerRunningInfoTest.java | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfoTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfoTest.java b/common/src/test/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfoTest.java new file mode 100644 index 0000000000..b37189383a --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfoTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.junit.Before; +import org.junit.Test; + +import java.util.Properties; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.apache.rocketmq.common.protocol.heartbeat.ConsumeType.CONSUME_ACTIVELY; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ConsumerRunningInfoTest { + + private ConsumerRunningInfo consumerRunningInfo; + + private TreeMap criTable; + + private MessageQueue messageQueue; + + @Before + public void init() { + consumerRunningInfo = new ConsumerRunningInfo(); + consumerRunningInfo.setJstack("test"); + + TreeMap mqTable = new TreeMap(); + messageQueue = new MessageQueue("topicA","broker", 1); + mqTable.put(messageQueue, new ProcessQueueInfo()); + consumerRunningInfo.setMqTable(mqTable); + + TreeMap statusTable = new TreeMap(); + statusTable.put("topicA", new ConsumeStatus()); + consumerRunningInfo.setStatusTable(statusTable); + + TreeSet subscriptionSet = new TreeSet(); + subscriptionSet.add(new SubscriptionData()); + consumerRunningInfo.setSubscriptionSet(subscriptionSet); + + Properties properties = new Properties(); + properties.put(ConsumerRunningInfo.PROP_CONSUME_TYPE, CONSUME_ACTIVELY); + properties.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, System.currentTimeMillis()); + consumerRunningInfo.setProperties(properties); + + criTable = new TreeMap(); + criTable.put("client_id", consumerRunningInfo); + } + + @Test + public void testFromJson() { + String toJson = RemotingSerializable.toJson(consumerRunningInfo, true); + ConsumerRunningInfo fromJson = RemotingSerializable.fromJson(toJson, ConsumerRunningInfo.class); + + assertThat(fromJson.getJstack()).isEqualTo("test"); + assertThat(fromJson.getProperties().get(ConsumerRunningInfo.PROP_CONSUME_TYPE)).isEqualTo(ConsumeType.CONSUME_ACTIVELY.name()); + + ConsumeStatus consumeStatus = fromJson.getStatusTable().get("topicA"); + assertThat(consumeStatus).isExactlyInstanceOf(ConsumeStatus.class); + + SubscriptionData subscription = fromJson.getSubscriptionSet().first(); + assertThat(subscription).isExactlyInstanceOf(SubscriptionData.class); + + ProcessQueueInfo processQueueInfo = fromJson.getMqTable().get(messageQueue); + assertThat(processQueueInfo).isExactlyInstanceOf(ProcessQueueInfo.class); + } + + @Test + public void testAnalyzeRebalance(){ + boolean result = ConsumerRunningInfo.analyzeRebalance(criTable); + assertThat(result).isTrue(); + } + + @Test + public void testAnalyzeProcessQueue(){ + String result = ConsumerRunningInfo.analyzeProcessQueue("client_id", consumerRunningInfo); + assertThat(result).isEmpty(); + + } + + @Test + public void testAnalyzeSubscription(){ + boolean result = ConsumerRunningInfo.analyzeSubscription(criTable); + assertThat(result).isTrue(); + } + + +} From a1d4ce01414c41dd97d54010a1ff129b0d9d46d0 Mon Sep 17 00:00:00 2001 From: Jlcao Date: Sat, 9 Mar 2019 14:20:49 +0800 Subject: [PATCH 118/122] [RIP-9](Operation) within deleted title(#1066) --- docs/cn/operation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/operation.md b/docs/cn/operation.md index c00be933a7..ecd37079d0 100644 --- a/docs/cn/operation.md +++ b/docs/cn/operation.md @@ -1,4 +1,4 @@ -## 运维管理(operation） +## 运维管理 ### 1 集群搭建 From 5134a8e93419c600eb86d2e6572f9d404d73eed9 Mon Sep 17 00:00:00 2001 From: github-ygy Date: Sat, 9 Mar 2019 14:22:20 +0800 Subject: [PATCH 119/122] [PIR-10] add test for GroupList (#807) --- .../common/protocol/GroupListTest.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 common/src/test/java/org/apache/rocketmq/common/protocol/GroupListTest.java diff --git a/common/src/test/java/org/apache/rocketmq/common/protocol/GroupListTest.java b/common/src/test/java/org/apache/rocketmq/common/protocol/GroupListTest.java new file mode 100644 index 0000000000..854ff05ddf --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/protocol/GroupListTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol; + +import org.apache.rocketmq.common.protocol.body.GroupList; +import org.junit.Test; + +import java.util.HashSet; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Created by guoyao on 2019/2/18. + */ +public class GroupListTest { + + @Test + public void testSetGet() throws Exception { + HashSet fisrtUniqueSet=createUniqueNewSet(); + HashSet secondUniqueSet=createUniqueNewSet(); + assertThat(fisrtUniqueSet).isNotEqualTo(secondUniqueSet); + GroupList gl=new GroupList(); + gl.setGroupList(fisrtUniqueSet); + assertThat(gl.getGroupList()).isEqualTo(fisrtUniqueSet); + assertThat(gl.getGroupList()).isNotEqualTo(secondUniqueSet); + gl.setGroupList(secondUniqueSet); + assertThat(gl.getGroupList()).isNotEqualTo(fisrtUniqueSet); + assertThat(gl.getGroupList()).isEqualTo(secondUniqueSet); + } + + private HashSet createUniqueNewSet() { + HashSet groups=new HashSet(); + groups.add(UUID.randomUUID().toString()); + return groups; + } +} From a974ed28670468da396b97cd978a64705722cf17 Mon Sep 17 00:00:00 2001 From: ly <641921791@qq.com> Date: Sun, 10 Mar 2019 12:25:44 +0800 Subject: [PATCH 120/122] [RIP-9] Modify the numbering style --- docs/cn/RocketMQ_Example.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index cfd78ea405..885b00a340 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -639,9 +639,9 @@ consumer.start(); 事务消息共有三种状态，提交状态、回滚状态、中间状态： -1. TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。 -2. TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。 -3. TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。 +- TransactionStatus.CommitTransaction: 提交事务，它允许消费者消费此消息。 +- TransactionStatus.RollbackTransaction: 回滚事务，它代表该消息将被删除，不允许被消费。 +- TransactionStatus.Unknown: 中间状态，它代表需要检查消息队列来确定状态。 ### 6.1 发送事务消息样例 From 0d8c0ba29f91fe53870e95cb247904587e6dedfa Mon Sep 17 00:00:00 2001 From: RickySong <761041702@qq.com> Date: Sun, 10 Mar 2019 12:44:50 +0800 Subject: [PATCH 121/122] Update Design_Store.md --- docs/en/Design_Store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Design_Store.md b/docs/en/Design_Store.md index d0d1c90aaa..cbc661305d 100644 --- a/docs/en/Design_Store.md +++ b/docs/en/Design_Store.md @@ -15,7 +15,7 @@ The message storage architecture diagram consists of 3 files related to message * `CommitLog`：The `CommitLog` file stores message body and metadata sent by producer, and the message content is not fixed length. The default size of one `CommitLog` file is 1G, the length of the file name is 20 digits, the left side is zero padded, and the remaining is the starting offset. For example, `00000000000000000000` represents the first file, the starting offset is 0, and the file size is 1G=1073741824, when the first `CommitLog` file is full, the second `CommitLog` file is `00000000001073741824`, the starting offset is 1073741824, and so on. The message is mainly appended to the log file sequentially. When one `CommitLog` file is full, the next will be written. -* `ConsumeQueue`: The `ConsumeQueue` is used to improve the performance of message consumption. Since RocketMQ uses topic-based subscription mode, message consumption is specific to the topic. Traversing the commitlog file to retrieve messages of one topic is very inefficient. The consumer can find the messages to be consumed according to the `ConsumeQueue`. The `ConsumeQueue`(logic consume queue) as an index of the consuming message stores the starting physical offset `offset` in `CommitLog` of the specified topic, the message size `size` and the hash code of the message tag. The `ConsumeQueue` file can be regarded as a topic-based `CommitLog` index file, so the consumequeue folder is organized as follows: `topic/queue/file` three-layer organization structure, the specific storage path is `$HOME/store/consumequeue/{topic}/{queueId }/{fileName}`. The consumequeue file uses a fixed-length design, each entry occupies 20 bytes, which is an 8-byte commitlog physical offset, a 4-byte message length, and an 8-byte tag hashcode. One consumequeue file consists of 30W entries, each entry can be randomly accessed like an array, each `ConsumeQueue` file's size is about 5.72M. +* `ConsumeQueue`: The `ConsumeQueue` is used to improve the performance of message consumption. Since RocketMQ uses topic-based subscription mode, message consumption is specific to the topic. Traversing the commitlog file to retrieve messages of one topic is very inefficient. The consumer can find the messages to be consumed according to the `ConsumeQueue`. The `ConsumeQueue`(logic consume queue) as an index of the consuming message stores the starting physical offset `offset` in `CommitLog` of the specified topic, the message size `size` and the hash code of the message tag. The `ConsumeQueue` file can be regarded as a topic-based `CommitLog` index file, so the consumequeue folder is organized as follows: `topic/queue/file` three-layer organization structure, the specific storage path is `$HOME/store/consumequeue/{topic}/{queueId }/{fileName}`. The consumequeue file uses a fixed-length design, each entry occupies 20 bytes, which is an 8-byte commitlog physical offset, a 4-byte message length, and an 8-byte tag hashcode. One consumequeue file consists of 0.3 million entries, each entry can be randomly accessed like an array, each `ConsumeQueue` file's size is about 5.72MB. * `IndexFile`: The `IndexFile` provides a way to query messages by key or time interval. The path of the `IndexFile` is `$HOME/store/index/${fileName}`, the file name `fileName` is named after the timestamp when it was created. One IndexFile's size is about 400M, and it can store 2000W indexes. The underlying storage of `IndexFile` is designed to implement the `HashMap` structure in the file system, so RocketMQ's index file is a hash index. From cf2a40bffd33af5f217a4e36af6e84b80aab0cd5 Mon Sep 17 00:00:00 2001 From: JJ Date: Mon, 11 Mar 2019 17:35:03 +0800 Subject: [PATCH 122/122] [RIP-9] Modify English document for Design_Trancation.md --- docs/en/Design_Trancation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/Design_Trancation.md b/docs/en/Design_Trancation.md index e5f10ed75d..71cdabea29 100644 --- a/docs/en/Design_Trancation.md +++ b/docs/en/Design_Trancation.md @@ -36,7 +36,7 @@ The compensation phase is used to resolve the timeout or failure case of the mes 3. How Op message stored and the correspondence between Op message and Half message - RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation. + RocketMQ writes the Op message to a specific system topic(RMQ_SYS_TRANS_OP_HALF_TOPIC) which will be created via the method - TransactionalMessageUtil.buildOpTopic(); this topic is an internal Topic (like the topic of RMQ_SYS_TRANS_HALF_TOPIC) and will not be consumed by the user. The content of the Op message is the physical offset of the corresponding Half message. Through the Op message we can index to the Half message for subsequent check-back operation. ![](../cn/image/rocketmq_design_12.png)