diff --git a/common/src/main/java/org/apache/rocketmq/common/AbstractBrokerRunnable.java b/common/src/main/java/org/apache/rocketmq/common/AbstractBrokerRunnable.java new file mode 100644 index 0000000000..89b780a49d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/AbstractBrokerRunnable.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common; + +import org.apache.rocketmq.logging.InnerLoggerFactory; + +public abstract class AbstractBrokerRunnable implements Runnable { + protected final BrokerIdentity brokerIdentity; + + public AbstractBrokerRunnable(BrokerIdentity brokerIdentity) { + this.brokerIdentity = brokerIdentity; + } + + /** + * real logic for running + */ + public abstract void run2(); + + @Override + public void run() { + if (brokerIdentity.isInBrokerContainer()) { + // set threadlocal broker identity to forward logging to corresponding broker + InnerLoggerFactory.brokerIdentity.set(brokerIdentity.getCanonicalName()); + } + run2(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 66fd4496e1..efd4ec90ae 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -16,8 +16,6 @@ */ package org.apache.rocketmq.common; -import java.net.InetAddress; -import java.net.UnknownHostException; import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; @@ -27,21 +25,25 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingUtil; -public class BrokerConfig { +public class BrokerConfig extends BrokerIdentity { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + private String brokerConfigPath = null; + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); @ImportantField private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, System.getenv(MixAll.NAMESRV_ADDR_ENV)); + + /** + * Listen port for single broker + */ + @ImportantField + private int listenPort = 6888; + @ImportantField private String brokerIP1 = RemotingUtil.getLocalAddress(); private String brokerIP2 = RemotingUtil.getLocalAddress(); - @ImportantField - private String brokerName = localHostName(); - @ImportantField - private String brokerClusterName = "DefaultCluster"; - @ImportantField - private long brokerId = MixAll.MASTER_ID; + private int brokerPermission = PermName.PERM_READ | PermName.PERM_WRITE; private int defaultTopicQueueNums = 8; @ImportantField @@ -53,6 +55,8 @@ public class BrokerConfig { @ImportantField private boolean autoCreateSubscriptionGroup = true; private String messageStorePlugIn = ""; + + private static final int PROCESSOR_NUMBER = Runtime.getRuntime().availableProcessors(); @ImportantField private String msgTraceTopicName = TopicValidator.RMQ_SYS_TRACE_TOPIC; @ImportantField @@ -60,24 +64,25 @@ public class BrokerConfig { /** * thread numbers for send message thread pool. */ - private int sendMessageThreadPoolNums = Math.min(Runtime.getRuntime().availableProcessors(), 4); - private int putMessageFutureThreadPoolNums = Math.min(Runtime.getRuntime().availableProcessors(), 4); - private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; + private int sendMessageThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4); + private int putMessageFutureThreadPoolNums = Math.min(PROCESSOR_NUMBER, 4); + private int pullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; + private int litePullMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; private int ackMessageThreadPoolNums = 3; - private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; - private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors(); + private int processReplyMessageThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; + private int queryMessageThreadPoolNums = 8 + PROCESSOR_NUMBER; private int adminBrokerThreadPoolNums = 16; private int clientManageThreadPoolNums = 32; private int consumerManageThreadPoolNums = 32; private int loadBalanceProcessorThreadPoolNums = 32; - private int heartbeatThreadPoolNums = Math.min(32, Runtime.getRuntime().availableProcessors()); + private int heartbeatThreadPoolNums = Math.min(32, PROCESSOR_NUMBER); /** * Thread numbers for EndTransactionProcessor */ - private int endTransactionThreadPoolNums = Math.max(8 + Runtime.getRuntime().availableProcessors() * 2, - sendMessageThreadPoolNums * 4); + private int endTransactionThreadPoolNums = Math.max(8 + PROCESSOR_NUMBER * 2, + sendMessageThreadPoolNums * 4); private int flushConsumerOffsetInterval = 1000 * 5; @@ -90,6 +95,7 @@ public class BrokerConfig { private int sendThreadPoolQueueCapacity = 10000; private int putThreadPoolQueueCapacity = 10000; private int pullThreadPoolQueueCapacity = 100000; + private int litePullThreadPoolQueueCapacity = 100000; private int ackThreadPoolQueueCapacity = 100000; private int replyThreadPoolQueueCapacity = 10000; private int queryThreadPoolQueueCapacity = 20000; @@ -97,6 +103,8 @@ public class BrokerConfig { private int consumerManagerThreadPoolQueueCapacity = 1000000; private int heartbeatThreadPoolQueueCapacity = 50000; private int endTransactionPoolQueueCapacity = 100000; + private int adminBrokerThreadPoolQueueCapacity = 10000; + private int loadBalanceThreadPoolQueueCapacity = 100000; private int filterServerNums = 0; @@ -108,17 +116,20 @@ public class BrokerConfig { private boolean highSpeedMode = false; - private boolean commercialEnable = true; - private int commercialTimerCount = 1; - private int commercialTransCount = 1; - private int commercialBigCount = 1; private int commercialBaseCount = 1; + private int commercialSizePerMsg = 4 * 1024; + + private boolean accountStatsEnable = true; + private boolean accountStatsPrintZeroValues = true; + private boolean transferMsgByHeap = true; private int maxDelayTime = 40; private String regionId = MixAll.DEFAULT_TRACE_REGION_ID; - private int registerBrokerTimeoutMills = 6000; + private int registerBrokerTimeoutMills = 24000; + + private int sendHeartbeatTimeoutMillis = 1000; private boolean slaveReadEnable = false; @@ -128,8 +139,10 @@ public class BrokerConfig { private boolean brokerFastFailureEnable = true; private long waitTimeMillsInSendQueue = 200; private long waitTimeMillsInPullQueue = 5 * 1000; + private long waitTimeMillsInLitePullQueue = 5 * 1000; private long waitTimeMillsInHeartbeatQueue = 31 * 1000; private long waitTimeMillsInTransactionQueue = 3 * 1000; + private long waitTimeMillsInAckQueue = 3000; private long startAcceptSendRequestTimeStamp = 0L; @@ -141,6 +154,9 @@ public class BrokerConfig { // 2. Filter bit map will be saved to consume queue extend file if allowed. private boolean enableCalcFilterBitMap = false; + //Reject the pull consumer instance to pull messages from broker. + private boolean rejectPullConsumerEnable = false; + // Expect num of consumers will use filter. private int expectConsumerNumUseFilter = 32; @@ -160,10 +176,23 @@ public class BrokerConfig { /** * This configurable item defines interval of topics registration of broker to name server. Allowing values are - * between 10, 000 and 60, 000 milliseconds. + * between 10,000 and 60,000 milliseconds. */ private int registerNameServerPeriod = 1000 * 30; + /** + * the interval to send heartbeat to name server for liveness detection. + */ + private int brokerHeartbeatInterval = 1000; + + /** + * How long the broker will be considered as inactive by nameserver since last heartbeat. Effective only if + * enableSlaveActingMaster is true + */ + private long brokerNotActiveTimeoutMillis = 10 * 1000; + + private boolean netWorkFlowController = true; + private int popPollingSize = 1024; private int popPollingMapSize = 100000; // 20w cost 200M heap memory. @@ -172,13 +201,20 @@ public class BrokerConfig { private long reviveInterval = 1000; private long reviveMaxSlow = 3; private long reviveScanTime = 10000; - private boolean enablePopLog = true; + private boolean enablePopLog = false; private boolean enablePopBufferMerge = false; private int popCkStayBufferTime = 10 * 1000; private int popCkStayBufferTimeOut = 3 * 1000; private int popCkMaxBufferSize = 200000; private int popCkOffsetMaxQueueSize = 20000; + private boolean realTimeNotifyConsumerChange = true; + + private boolean litePullMessageEnable = true; + + // The period to sync broker member group from namesrv, default value is 1 second + private int syncBrokerMemberGroupPeriod = 1000; + /** * the interval of pulling topic information from the named server */ @@ -233,15 +269,31 @@ public class BrokerConfig { private long forwardTimeout = 3 * 1000; - public static String localHostName() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - log.error("Failed to obtain the host name", e); - } + /** + * Slave will act master when failover. For example, if master down, timer or transaction message which is expire in slave will + * put to master (master of the same process in broker container mode or other masters in cluster when enableFailoverRemotingActing is true) + * when enableSlaveActingMaster is true + */ + private boolean enableSlaveActingMaster = false; - return "DEFAULT_BROKER"; - } + private boolean enableRemoteEscape = false; + + private boolean skipPreOnline = false; + + private boolean asyncSendEnable = true; + + private long consumerOffsetUpdateVersionStep = 500; + + private long delayOffsetUpdateVersionStep = 200; + + /** + * Whether to lock quorum replicas. + * + * True: need to lock quorum replicas succeed. False: only need to lock one replica succeed. + */ + private boolean lockInStrictMode = false; + + private String metaDataHosts = ""; public long getMaxPopPollingSize() { return maxPopPollingSize; @@ -451,22 +503,6 @@ public class BrokerConfig { this.highSpeedMode = highSpeedMode; } - public String getRocketmqHome() { - return rocketmqHome; - } - - public void setRocketmqHome(String rocketmqHome) { - this.rocketmqHome = rocketmqHome; - } - - public String getBrokerName() { - return brokerName; - } - - public void setBrokerName(String brokerName) { - this.brokerName = brokerName; - } - public int getBrokerPermission() { return brokerPermission; } @@ -491,14 +527,6 @@ public class BrokerConfig { this.autoCreateTopicEnable = autoCreateTopic; } - public String getBrokerClusterName() { - return brokerClusterName; - } - - public void setBrokerClusterName(String brokerClusterName) { - this.brokerClusterName = brokerClusterName; - } - public String getBrokerIP1() { return brokerIP1; } @@ -603,14 +631,6 @@ public class BrokerConfig { this.namesrvAddr = namesrvAddr; } - public long getBrokerId() { - return brokerId; - } - - public void setBrokerId(long brokerId) { - this.brokerId = brokerId; - } - public boolean isAutoCreateSubscriptionGroup() { return autoCreateSubscriptionGroup; } @@ -619,6 +639,94 @@ public class BrokerConfig { this.autoCreateSubscriptionGroup = autoCreateSubscriptionGroup; } + public String getBrokerConfigPath() { + return brokerConfigPath; + } + + public void setBrokerConfigPath(String brokerConfigPath) { + this.brokerConfigPath = brokerConfigPath; + } + + public String getRocketmqHome() { + return rocketmqHome; + } + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + public int getListenPort() { + return listenPort; + } + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + public int getLitePullMessageThreadPoolNums() { + return litePullMessageThreadPoolNums; + } + + public void setLitePullMessageThreadPoolNums(int litePullMessageThreadPoolNums) { + this.litePullMessageThreadPoolNums = litePullMessageThreadPoolNums; + } + + public int getLitePullThreadPoolQueueCapacity() { + return litePullThreadPoolQueueCapacity; + } + + public void setLitePullThreadPoolQueueCapacity(int litePullThreadPoolQueueCapacity) { + this.litePullThreadPoolQueueCapacity = litePullThreadPoolQueueCapacity; + } + + public int getAdminBrokerThreadPoolQueueCapacity() { + return adminBrokerThreadPoolQueueCapacity; + } + + public void setAdminBrokerThreadPoolQueueCapacity(int adminBrokerThreadPoolQueueCapacity) { + this.adminBrokerThreadPoolQueueCapacity = adminBrokerThreadPoolQueueCapacity; + } + + public int getLoadBalanceThreadPoolQueueCapacity() { + return loadBalanceThreadPoolQueueCapacity; + } + + public void setLoadBalanceThreadPoolQueueCapacity(int loadBalanceThreadPoolQueueCapacity) { + this.loadBalanceThreadPoolQueueCapacity = loadBalanceThreadPoolQueueCapacity; + } + + public int getSendHeartbeatTimeoutMillis() { + return sendHeartbeatTimeoutMillis; + } + + public void setSendHeartbeatTimeoutMillis(int sendHeartbeatTimeoutMillis) { + this.sendHeartbeatTimeoutMillis = sendHeartbeatTimeoutMillis; + } + + public long getWaitTimeMillsInLitePullQueue() { + return waitTimeMillsInLitePullQueue; + } + + public void setWaitTimeMillsInLitePullQueue(long waitTimeMillsInLitePullQueue) { + this.waitTimeMillsInLitePullQueue = waitTimeMillsInLitePullQueue; + } + + public boolean isLitePullMessageEnable() { + return litePullMessageEnable; + } + + public void setLitePullMessageEnable(boolean litePullMessageEnable) { + this.litePullMessageEnable = litePullMessageEnable; + } + + public int getSyncBrokerMemberGroupPeriod() { + return syncBrokerMemberGroupPeriod; + } + + public void setSyncBrokerMemberGroupPeriod(int syncBrokerMemberGroupPeriod) { + this.syncBrokerMemberGroupPeriod = syncBrokerMemberGroupPeriod; + } + public boolean isRejectTransactionMessage() { return rejectTransactionMessage; } @@ -731,38 +839,6 @@ public class BrokerConfig { this.clientManageThreadPoolNums = clientManageThreadPoolNums; } - public boolean isCommercialEnable() { - return commercialEnable; - } - - public void setCommercialEnable(final boolean commercialEnable) { - this.commercialEnable = commercialEnable; - } - - public int getCommercialTimerCount() { - return commercialTimerCount; - } - - public void setCommercialTimerCount(final int commercialTimerCount) { - this.commercialTimerCount = commercialTimerCount; - } - - public int getCommercialTransCount() { - return commercialTransCount; - } - - public void setCommercialTransCount(final int commercialTransCount) { - this.commercialTransCount = commercialTransCount; - } - - public int getCommercialBigCount() { - return commercialBigCount; - } - - public void setCommercialBigCount(final int commercialBigCount) { - this.commercialBigCount = commercialBigCount; - } - public int getMaxDelayTime() { return maxDelayTime; } @@ -1051,6 +1127,142 @@ public class BrokerConfig { this.forwardTimeout = timeout; } + public int getBrokerHeartbeatInterval() { + return brokerHeartbeatInterval; + } + + public void setBrokerHeartbeatInterval(int brokerHeartbeatInterval) { + this.brokerHeartbeatInterval = brokerHeartbeatInterval; + } + + public long getBrokerNotActiveTimeoutMillis() { + return brokerNotActiveTimeoutMillis; + } + + public void setBrokerNotActiveTimeoutMillis(long brokerNotActiveTimeoutMillis) { + this.brokerNotActiveTimeoutMillis = brokerNotActiveTimeoutMillis; + } + + public boolean isNetWorkFlowController() { + return netWorkFlowController; + } + + public void setNetWorkFlowController(boolean netWorkFlowController) { + this.netWorkFlowController = netWorkFlowController; + } + + public boolean isRealTimeNotifyConsumerChange() { + return realTimeNotifyConsumerChange; + } + + public void setRealTimeNotifyConsumerChange(boolean realTimeNotifyConsumerChange) { + this.realTimeNotifyConsumerChange = realTimeNotifyConsumerChange; + } + + public boolean isEnableSlaveActingMaster() { + return enableSlaveActingMaster; + } + + public void setEnableSlaveActingMaster(boolean enableSlaveActingMaster) { + this.enableSlaveActingMaster = enableSlaveActingMaster; + } + + public boolean isEnableRemoteEscape() { + return enableRemoteEscape; + } + + public void setEnableRemoteEscape(boolean enableRemoteEscape) { + this.enableRemoteEscape = enableRemoteEscape; + } + + public boolean isSkipPreOnline() { + return skipPreOnline; + } + + public void setSkipPreOnline(boolean skipPreOnline) { + this.skipPreOnline = skipPreOnline; + } + + public boolean isAsyncSendEnable() { + return asyncSendEnable; + } + + public void setAsyncSendEnable(boolean asyncSendEnable) { + this.asyncSendEnable = asyncSendEnable; + } + + public long getConsumerOffsetUpdateVersionStep() { + return consumerOffsetUpdateVersionStep; + } + + public void setConsumerOffsetUpdateVersionStep(long consumerOffsetUpdateVersionStep) { + this.consumerOffsetUpdateVersionStep = consumerOffsetUpdateVersionStep; + } + + public long getDelayOffsetUpdateVersionStep() { + return delayOffsetUpdateVersionStep; + } + + public void setDelayOffsetUpdateVersionStep(long delayOffsetUpdateVersionStep) { + this.delayOffsetUpdateVersionStep = delayOffsetUpdateVersionStep; + } + + public int getCommercialSizePerMsg() { + return commercialSizePerMsg; + } + + public void setCommercialSizePerMsg(int commercialSizePerMsg) { + this.commercialSizePerMsg = commercialSizePerMsg; + } + + public long getWaitTimeMillsInAckQueue() { + return waitTimeMillsInAckQueue; + } + + public void setWaitTimeMillsInAckQueue(long waitTimeMillsInAckQueue) { + this.waitTimeMillsInAckQueue = waitTimeMillsInAckQueue; + } + + public boolean isRejectPullConsumerEnable() { + return rejectPullConsumerEnable; + } + + public void setRejectPullConsumerEnable(boolean rejectPullConsumerEnable) { + this.rejectPullConsumerEnable = rejectPullConsumerEnable; + } + + public boolean isAccountStatsEnable() { + return accountStatsEnable; + } + + public void setAccountStatsEnable(boolean accountStatsEnable) { + this.accountStatsEnable = accountStatsEnable; + } + + public boolean isAccountStatsPrintZeroValues() { + return accountStatsPrintZeroValues; + } + + public void setAccountStatsPrintZeroValues(boolean accountStatsPrintZeroValues) { + this.accountStatsPrintZeroValues = accountStatsPrintZeroValues; + } + + public boolean isLockInStrictMode() { + return lockInStrictMode; + } + + public void setLockInStrictMode(boolean lockInStrictMode) { + this.lockInStrictMode = lockInStrictMode; + } + + public String getMetaDataHosts() { + return metaDataHosts; + } + + public void setMetaDataHosts(String metaDataHosts) { + this.metaDataHosts = metaDataHosts; + } + public boolean isIsolateLogEnable() { return isolateLogEnable; } diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerIdentity.java b/common/src/main/java/org/apache/rocketmq/common/BrokerIdentity.java new file mode 100644 index 0000000000..e5ef3d7acc --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerIdentity.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.rocketmq.common.annotation.ImportantField; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InnerLoggerFactory; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class BrokerIdentity { + private static final String DEFAULT_CLUSTER_NAME = "DefaultCluster"; + protected static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + public static final BrokerIdentity BROKER_CONTAINER_IDENTITY = new BrokerIdentity(true); + + @ImportantField + private String brokerName = localHostName(); + @ImportantField + private String brokerClusterName = DEFAULT_CLUSTER_NAME; + @ImportantField + private volatile long brokerId = MixAll.MASTER_ID; + + private boolean isBrokerContainer = false; + + // Do not set it manually, it depends on the startup mode + // Broker start by BrokerStartup is false, start or add by BrokerContainer is true + private boolean isInBrokerContainer = false; + + public BrokerIdentity() { + } + + public BrokerIdentity(boolean isBrokerContainer) { + this.isBrokerContainer = isBrokerContainer; + } + + public BrokerIdentity(String brokerClusterName, String brokerName, long brokerId) { + this.brokerName = brokerName; + this.brokerClusterName = brokerClusterName; + this.brokerId = brokerId; + } + + public BrokerIdentity(String brokerClusterName, String brokerName, long brokerId, boolean isInBrokerContainer) { + this.brokerName = brokerName; + this.brokerClusterName = brokerClusterName; + this.brokerId = brokerId; + this.isInBrokerContainer = isInBrokerContainer; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(final String brokerName) { + this.brokerName = brokerName; + } + + public String getBrokerClusterName() { + return brokerClusterName; + } + + public void setBrokerClusterName(final String brokerClusterName) { + this.brokerClusterName = brokerClusterName; + } + + public long getBrokerId() { + return brokerId; + } + + public void setBrokerId(final long brokerId) { + this.brokerId = brokerId; + } + + public boolean isInBrokerContainer() { + return isInBrokerContainer; + } + + public void setInBrokerContainer(boolean inBrokerContainer) { + isInBrokerContainer = inBrokerContainer; + } + + protected static String localHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + LOGGER.error("Failed to obtain the host name", e); + } + + return "DEFAULT_BROKER"; + } + + public String getCanonicalName() { + if (isBrokerContainer) { + return InnerLoggerFactory.BROKER_CONTAINER_NAME; + } + return this.getBrokerClusterName() + "_" + this.getBrokerName() + "_" + this.getBrokerId(); + } + + public String getLoggerIdentifier() { + return "#" + getCanonicalName() + "#"; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final BrokerIdentity identity = (BrokerIdentity) o; + + return new EqualsBuilder() + .append(brokerId, identity.brokerId) + .append(brokerName, identity.brokerName) + .append(brokerClusterName, identity.brokerClusterName) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(brokerName) + .append(brokerClusterName) + .append(brokerId) + .toHashCode(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerSyncInfo.java b/common/src/main/java/org/apache/rocketmq/common/BrokerSyncInfo.java new file mode 100644 index 0000000000..1ec4bc5519 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerSyncInfo.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class BrokerSyncInfo extends RemotingSerializable { + /** + * For slave online sync, retrieve HA address before register + */ + private String masterHaAddress; + + private long masterFlushOffset; + + private String masterAddress; + + public BrokerSyncInfo(String masterHaAddress, long masterFlushOffset, String masterAddress) { + this.masterHaAddress = masterHaAddress; + this.masterFlushOffset = masterFlushOffset; + this.masterAddress = masterAddress; + } + + public String getMasterHaAddress() { + return masterHaAddress; + } + + public void setMasterHaAddress(String masterHaAddress) { + this.masterHaAddress = masterHaAddress; + } + + public long getMasterFlushOffset() { + return masterFlushOffset; + } + + public void setMasterFlushOffset(long masterFlushOffset) { + this.masterFlushOffset = masterFlushOffset; + } + + public String getMasterAddress() { + return masterAddress; + } + + public void setMasterAddress(String masterAddress) { + this.masterAddress = masterAddress; + } + + @Override + public String toString() { + return "BrokerSyncInfo{" + + "masterHaAddress='" + masterHaAddress + '\'' + + ", masterFlushOffset=" + masterFlushOffset + + ", masterAddress=" + masterAddress + + '}'; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/Configuration.java b/common/src/main/java/org/apache/rocketmq/common/Configuration.java index 064b5e61d1..29dafa1dc2 100644 --- a/common/src/main/java/org/apache/rocketmq/common/Configuration.java +++ b/common/src/main/java/org/apache/rocketmq/common/Configuration.java @@ -55,6 +55,9 @@ public class Configuration { return; } for (Object configObject : configObjects) { + if (configObject == null) { + continue; + } registerConfig(configObject); } } @@ -234,6 +237,24 @@ public class Configuration { return null; } + public String getClientConfigsFormatString(List clientKeys) { + try { + readWriteLock.readLock().lockInterruptibly(); + + try { + + return getClientConfigsInternal(clientKeys); + + } finally { + readWriteLock.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("getAllConfigsFormatString lock error"); + } + + return null; + } + public String getDataVersionJson() { return this.dataVersion.toJson(); } @@ -276,6 +297,26 @@ public class Configuration { return stringBuilder.toString(); } + private String getClientConfigsInternal(List clientConigKeys) { + StringBuilder stringBuilder = new StringBuilder(); + Properties clientProperties = new Properties(); + + // reload from config object ? + for (Object configObject : this.configObjectList) { + Properties properties = MixAll.object2Properties(configObject); + + for (String nameNow : clientConigKeys) { + if (properties.containsKey(nameNow)) { + clientProperties.put(nameNow, properties.get(nameNow)); + } + } + + } + stringBuilder.append(MixAll.properties2String(clientProperties)); + + return stringBuilder.toString(); + } + private void merge(Properties from, Properties to) { for (Entry next : from.entrySet()) { Object fromObj = next.getValue(), toObj = to.get(next.getKey()); diff --git a/common/src/main/java/org/apache/rocketmq/common/DataVersion.java b/common/src/main/java/org/apache/rocketmq/common/DataVersion.java index e54000deb6..6d53bd8214 100644 --- a/common/src/main/java/org/apache/rocketmq/common/DataVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/DataVersion.java @@ -20,19 +20,34 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class DataVersion extends RemotingSerializable { + private long stateVersion = 0L; private long timestamp = System.currentTimeMillis(); private AtomicLong counter = new AtomicLong(0); public void assignNewOne(final DataVersion dataVersion) { this.timestamp = dataVersion.timestamp; + this.stateVersion = dataVersion.stateVersion; this.counter.set(dataVersion.counter.get()); } public void nextVersion() { + this.nextVersion(0L); + } + + public void nextVersion(long stateVersion) { this.timestamp = System.currentTimeMillis(); + this.stateVersion = stateVersion; this.counter.incrementAndGet(); } + public long getStateVersion() { + return stateVersion; + } + + public void setStateVersion(long stateVersion) { + this.stateVersion = stateVersion; + } + public long getTimestamp() { return timestamp; } @@ -56,22 +71,25 @@ public class DataVersion extends RemotingSerializable { if (o == null || getClass() != o.getClass()) return false; - final DataVersion that = (DataVersion) o; + DataVersion version = (DataVersion) o; - if (timestamp != that.timestamp) { + if (getStateVersion() != version.getStateVersion()) return false; + if (getTimestamp() != version.getTimestamp()) + return false; + + if (counter != null && version.counter != null) { + return counter.longValue() == version.counter.longValue(); } - if (counter != null && that.counter != null) { - return counter.longValue() == that.counter.longValue(); - } + return (null == counter) && (null == version.counter); - return (null == counter) && (null == that.counter); } @Override public int hashCode() { - int result = (int) (timestamp ^ (timestamp >>> 32)); + int result = (int) (getStateVersion() ^ (getStateVersion() >>> 32)); + result = 31 * result + (int) (getTimestamp() ^ (getTimestamp() >>> 32)); if (null != counter) { long l = counter.get(); result = 31 * result + (int) (l ^ (l >>> 32)); @@ -87,4 +105,21 @@ public class DataVersion extends RemotingSerializable { sb.append(']'); return sb.toString(); } + + public int compare(DataVersion dataVersion) { + if (this.getStateVersion() > dataVersion.getStateVersion()) { + return 1; + } else if (this.getStateVersion() < dataVersion.getStateVersion()) { + return -1; + } else if (this.getCounter().get() > dataVersion.getCounter().get()) { + return 1; + } else if (this.getCounter().get() < dataVersion.getCounter().get()) { + return -1; + } else if (this.getTimestamp() > dataVersion.getTimestamp()) { + return 1; + } else if (this.getTimestamp() < dataVersion.getTimestamp()) { + return -1; + } + return 0; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/KeyBuilder.java b/common/src/main/java/org/apache/rocketmq/common/KeyBuilder.java index d30789f3dd..e1532d9399 100644 --- a/common/src/main/java/org/apache/rocketmq/common/KeyBuilder.java +++ b/common/src/main/java/org/apache/rocketmq/common/KeyBuilder.java @@ -34,4 +34,8 @@ public class KeyBuilder { public static String buildPollingKey(String topic, String cid, int queueId) { return topic + PopAckConstants.SPLIT + cid + PopAckConstants.SPLIT + queueId; } + + public static String buildPollingNotificationKey(String topic, int queueId) { + return topic + PopAckConstants.SPLIT + queueId; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/LockCallback.java b/common/src/main/java/org/apache/rocketmq/common/LockCallback.java new file mode 100644 index 0000000000..9abf36755d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/LockCallback.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +import java.util.Set; +import org.apache.rocketmq.common.message.MessageQueue; + +public interface LockCallback { + void onSuccess(final Set lockOKMQSet); + + void onException(final Throwable e); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index a15294fdc4..638d038069 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -74,7 +74,10 @@ public class MixAll { public static final String LOCALHOST = localhost(); public static final String DEFAULT_CHARSET = "UTF-8"; public static final long MASTER_ID = 0L; + public static final long FIRST_SLAVE_ID = 1L; public static final long CURRENT_JVM_PID = getPID(); + public final static int UNIT_PRE_SIZE_FOR_MSG = 28; + public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; public static final String REPLY_TOPIC_POSTFIX = "REPLY_TOPIC"; @@ -90,6 +93,7 @@ public class MixAll { public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = "__syslo__"; public static final String METADATA_SCOPE_GLOBAL = "__global__"; public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = "__syslo__none__"; + public static final String MULTI_PATH_SPLITTER = System.getProperty("rocketmq.broker.multiPathSplitter", ","); public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); diff --git a/common/src/main/java/org/apache/rocketmq/common/PopAckConstants.java b/common/src/main/java/org/apache/rocketmq/common/PopAckConstants.java index 839f947598..ac5a1a17ed 100644 --- a/common/src/main/java/org/apache/rocketmq/common/PopAckConstants.java +++ b/common/src/main/java/org/apache/rocketmq/common/PopAckConstants.java @@ -32,4 +32,13 @@ public class PopAckConstants { public static final String ACK_TAG = "ack"; public static final String SPLIT = "@"; + /** + * Build cluster revive topic + * + * @param clusterName cluster name + * @return revive topic + */ + public static String buildClusterReviveTopic(String clusterName) { + return PopAckConstants.REVIVE_TOPIC + clusterName; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index ead8117d6b..a84bcc861f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -27,7 +27,7 @@ public abstract class ServiceThread implements Runnable { private static final long JOIN_TIME = 90 * 1000; - private Thread thread; + protected Thread thread; protected final CountDownLatch2 waitPoint = new CountDownLatch2(1); protected volatile AtomicBoolean hasNotified = new AtomicBoolean(false); protected volatile boolean stopped = false; @@ -76,17 +76,17 @@ public abstract class ServiceThread implements Runnable { long beginTime = System.currentTimeMillis(); if (!this.thread.isDaemon()) { - this.thread.join(this.getJointime()); + this.thread.join(this.getJoinTime()); } long elapsedTime = System.currentTimeMillis() - beginTime; log.info("join thread " + this.getServiceName() + " elapsed time(ms) " + elapsedTime + " " - + this.getJointime()); + + this.getJoinTime()); } catch (InterruptedException e) { log.error("Interrupted", e); } } - public long getJointime() { + public long getJoinTime() { return JOIN_TIME; } diff --git a/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java b/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java index 564d60c54e..a76f605f79 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java +++ b/common/src/main/java/org/apache/rocketmq/common/ThreadFactoryImpl.java @@ -34,6 +34,19 @@ public class ThreadFactoryImpl implements ThreadFactory { this.daemon = daemon; } + public ThreadFactoryImpl(final String threadNamePrefix, BrokerIdentity brokerIdentity) { + this(threadNamePrefix, false, brokerIdentity); + } + + public ThreadFactoryImpl(final String threadNamePrefix, boolean daemon, BrokerIdentity brokerIdentity) { + this.daemon = daemon; + if (brokerIdentity != null && brokerIdentity.isInBrokerContainer()) { + this.threadNamePrefix = brokerIdentity.getLoggerIdentifier() + threadNamePrefix; + } else { + this.threadNamePrefix = threadNamePrefix; + } + } + @Override public Thread newThread(Runnable r) { Thread thread = new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet()); diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java index 610c3e28ae..f11b3b35ca 100644 --- a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java @@ -40,16 +40,6 @@ public class TopicConfig { this.topicName = topicName; } - public TopicConfig(TopicConfig other) { - this.topicName = other.topicName; - this.readQueueNums = other.readQueueNums; - this.writeQueueNums = other.writeQueueNums; - this.perm = other.perm; - this.topicFilterType = other.topicFilterType; - this.topicSysFlag = other.topicSysFlag; - this.order = other.order; - } - public TopicConfig(String topicName, int readQueueNums, int writeQueueNums) { this.topicName = topicName; this.readQueueNums = readQueueNums; @@ -63,16 +53,38 @@ public class TopicConfig { this.perm = perm; } + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm, int topicSysFlag) { + this.topicName = topicName; + this.readQueueNums = readQueueNums; + this.writeQueueNums = writeQueueNums; + this.perm = perm; + this.topicSysFlag = topicSysFlag; + } + + public TopicConfig(TopicConfig other) { + this.topicName = other.topicName; + this.readQueueNums = other.readQueueNums; + this.writeQueueNums = other.writeQueueNums; + this.perm = other.perm; + this.topicFilterType = other.topicFilterType; + this.topicSysFlag = other.topicSysFlag; + this.order = other.order; + } public String encode() { StringBuilder sb = new StringBuilder(); + //[0] sb.append(this.topicName); sb.append(SEPARATOR); + //[1] sb.append(this.readQueueNums); sb.append(SEPARATOR); + //[2] sb.append(this.writeQueueNums); sb.append(SEPARATOR); + //[3] sb.append(this.perm); sb.append(SEPARATOR); + //[4] sb.append(this.topicFilterType); // Leave the encode/decode [attributes] out for now @@ -82,7 +94,7 @@ public class TopicConfig { public boolean decode(final String in) { String[] strs = in.split(SEPARATOR); - if (strs != null && strs.length == 5) { + if (strs != null && strs.length >= 5) { this.topicName = strs[0]; this.readQueueNums = Integer.parseInt(strs[1]); @@ -165,18 +177,27 @@ public class TopicConfig { @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; TopicConfig that = (TopicConfig) o; - if (readQueueNums != that.readQueueNums) return false; - if (writeQueueNums != that.writeQueueNums) return false; - if (perm != that.perm) return false; - if (topicSysFlag != that.topicSysFlag) return false; - if (order != that.order) return false; - if (topicName != null ? !topicName.equals(that.topicName) : that.topicName != null) return false; - if (topicFilterType != that.topicFilterType) return false; + if (readQueueNums != that.readQueueNums) + return false; + if (writeQueueNums != that.writeQueueNums) + return false; + if (perm != that.perm) + return false; + if (topicSysFlag != that.topicSysFlag) + return false; + if (order != that.order) + return false; + if (topicName != null ? !topicName.equals(that.topicName) : that.topicName != null) + return false; + if (topicFilterType != that.topicFilterType) + return false; return attributes != null ? attributes.equals(that.attributes) : that.attributes == null; } diff --git a/common/src/main/java/org/apache/rocketmq/common/UnlockCallback.java b/common/src/main/java/org/apache/rocketmq/common/UnlockCallback.java new file mode 100644 index 0000000000..e86ec8b4af --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/UnlockCallback.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common; + +public interface UnlockCallback { + void onSuccess(); + + void onException(final Throwable e); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index a0d35ab8b9..772754420b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -21,10 +21,14 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; +import java.nio.ByteBuffer; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.text.NumberFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -229,7 +233,6 @@ public class UtilAll { return -1; } - long totalSpace = file.getTotalSpace(); if (totalSpace > 0) { @@ -251,6 +254,25 @@ public class UtilAll { return -1; } + public static long getDiskPartitionTotalSpace(final String path) { + if (null == path || path.isEmpty()) { + return -1; + } + + try { + File file = new File(path); + + + if (!file.exists()) { + return -1; + } + + return file.getTotalSpace() - file.getFreeSpace() + file.getUsableSpace(); + } catch (Exception e) { + return -1; + } + } + public static int crc32(byte[] array) { if (array != null) { return crc32(array, 0, array.length); @@ -619,4 +641,88 @@ public class UtilAll { String[] addrArray = str.split(splitter); return Arrays.asList(addrArray); } + + public static void deleteEmptyDirectory(File file) { + if (file == null || !file.exists()) { + return; + } + if (!file.isDirectory()) { + return; + } + File[] files = file.listFiles(); + if (files == null || files.length <= 0) { + file.delete(); + log.info("delete empty direct, {}", file.getPath()); + } + } + + public static void cleanBuffer(final ByteBuffer buffer) { + if (buffer == null || !buffer.isDirect() || buffer.capacity() == 0) { + return; + } + invoke(invoke(viewed(buffer), "cleaner"), "clean"); + } + + public static Object invoke(final Object target, final String methodName, final Class... args) { + return AccessController.doPrivileged(new PrivilegedAction() { + @Override + public Object run() { + try { + Method method = method(target, methodName, args); + method.setAccessible(true); + return method.invoke(target); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + }); + } + + public static Method method(Object target, String methodName, Class[] args) throws NoSuchMethodException { + try { + return target.getClass().getMethod(methodName, args); + } catch (NoSuchMethodException e) { + return target.getClass().getDeclaredMethod(methodName, args); + } + } + + private static ByteBuffer viewed(ByteBuffer buffer) { + String methodName = "viewedBuffer"; + + Method[] methods = buffer.getClass().getMethods(); + for (Method method : methods) { + if (method.getName().equals("attachment")) { + methodName = "attachment"; + break; + } + } + + ByteBuffer viewedBuffer = (ByteBuffer) invoke(buffer, methodName); + if (viewedBuffer == null) { + return buffer; + } else { + return viewed(viewedBuffer); + } + } + + public static void ensureDirOK(final String dirName) { + if (dirName != null) { + if (dirName.contains(MixAll.MULTI_PATH_SPLITTER)) { + String[] dirs = dirName.trim().split(MixAll.MULTI_PATH_SPLITTER); + for (String dir : dirs) { + createDirIfNotExist(dir); + } + } else { + createDirIfNotExist(dirName); + } + } + } + + private static void createDirIfNotExist(String dirName) { + File f = new File(dirName); + if (!f.exists()) { + boolean result = f.mkdirs(); + log.info(dirName + " mkdir " + (result ? "OK" : "Failed")); + } + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java index c29eccd4cb..a77d5c2d21 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java @@ -19,7 +19,7 @@ package org.apache.rocketmq.common.constant; public class LoggerName { public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; - public static final String NAMESRV_CONSOLE_NAME = "RocketmqNamesrvConsole"; + public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; @@ -30,7 +30,11 @@ public class LoggerName { public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; + public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; + public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; + public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; + public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; @@ -38,5 +42,6 @@ public class LoggerName { public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; + public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; public static final String STDOUT_LOGGER_NAME = "STDOUT"; } diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java index 1fe9aa106e..434870cbae 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java @@ -17,10 +17,16 @@ package org.apache.rocketmq.common.constant; public class PermName { - public static final int PERM_PRIORITY = 0x1 << 3; - public static final int PERM_READ = 0x1 << 2; - public static final int PERM_WRITE = 0x1 << 1; - public static final int PERM_INHERIT = 0x1; + public static final int INDEX_PERM_PRIORITY = 3; + public static final int INDEX_PERM_READ = 2; + public static final int INDEX_PERM_WRITE = 1; + public static final int INDEX_PERM_INHERIT = 0; + + + public static final int PERM_PRIORITY = 0x1 << INDEX_PERM_PRIORITY; + public static final int PERM_READ = 0x1 << INDEX_PERM_READ; + public static final int PERM_WRITE = 0x1 << INDEX_PERM_WRITE; + public static final int PERM_INHERIT = 0x1 << INDEX_PERM_INHERIT; public static String perm2String(final int perm) { final StringBuilder sb = new StringBuilder("---"); @@ -50,4 +56,8 @@ public class PermName { public static boolean isInherited(final int perm) { return (perm & PERM_INHERIT) == PERM_INHERIT; } + + public static boolean isPriority(final int perm) { + return (perm & PERM_PRIORITY) == PERM_PRIORITY; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/future/FutureTaskExt.java b/common/src/main/java/org/apache/rocketmq/common/future/FutureTaskExt.java new file mode 100644 index 0000000000..1bd46eafa2 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/future/FutureTaskExt.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.future; + +import java.util.concurrent.Callable; +import java.util.concurrent.FutureTask; + +public class FutureTaskExt extends FutureTask { + private final Runnable runnable; + + public FutureTaskExt(final Callable callable) { + super(callable); + this.runnable = null; + } + + public FutureTaskExt(final Runnable runnable, final V result) { + super(runnable, result); + this.runnable = runnable; + } + + public Runnable getRunnable() { + return runnable; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 105ef5b1b4..a823466415 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -39,13 +39,16 @@ public class MessageConst { public static final String PROPERTY_MSG_REGION = "MSG_REGION"; public static final String PROPERTY_TRACE_SWITCH = "TRACE_ON"; public static final String PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX = "UNIQ_KEY"; + public static final String PROPERTY_EXTEND_UNIQ_INFO = "EXTEND_UNIQ_INFO"; public static final String PROPERTY_MAX_RECONSUME_TIMES = "MAX_RECONSUME_TIMES"; public static final String PROPERTY_CONSUME_START_TIMESTAMP = "CONSUME_START_TIME"; public static final String PROPERTY_INNER_NUM = "INNER_NUM"; public static final String PROPERTY_INNER_BASE = "INNER_BASE"; - public static final String PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET = "TRAN_PREPARED_QUEUE_OFFSET"; - public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; + public static final String DUP_INFO = "DUP_INFO"; public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; + public static final String PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET = "TRAN_PREPARED_QUEUE_OFFSET"; + public static final String PROPERTY_TRANSACTION_ID = "__transactionId__"; + public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; public static final String PROPERTY_INSTANCE_ID = "INSTANCE_ID"; public static final String PROPERTY_CORRELATION_ID = "CORRELATION_ID"; public static final String PROPERTY_MESSAGE_REPLY_TO_CLIENT = "REPLY_TO_CLIENT"; @@ -55,12 +58,29 @@ public class MessageConst { public static final String PROPERTY_CLUSTER = "CLUSTER"; public static final String PROPERTY_MESSAGE_TYPE = "MSG_TYPE"; public static final String PROPERTY_POP_CK = "POP_CK"; + public static final String PROPERTY_POP_CK_OFFSET = "POP_CK_OFFSET"; public static final String PROPERTY_FIRST_POP_TIME = "1ST_POP_TIME"; + public static final String PROPERTY_SHARDING_KEY = "__SHARDINGKEY"; public static final String PROPERTY_FORWARD_QUEUE_ID = "PROPERTY_FORWARD_QUEUE_ID"; public static final String PROPERTY_REDIRECT = "REDIRECT"; public static final String PROPERTY_INNER_MULTI_DISPATCH = "INNER_MULTI_DISPATCH"; public static final String PROPERTY_INNER_MULTI_QUEUE_OFFSET = "INNER_MULTI_QUEUE_OFFSET"; + /** + * property which name starts with "__RMQ.TRANSIENT." is called transient one that will not stored in broker disks. + */ + public static final String PROPERTY_TRANSIENT_PREFIX = "__RMQ.TRANSIENT."; + + /** + * the transient property key of topicSysFlag (set by client when pulling messages) + */ + public static final String PROPERTY_TRANSIENT_TOPIC_CONFIG = PROPERTY_TRANSIENT_PREFIX + "TOPIC_SYS_FLAG"; + + /** + * the transient property key of groupSysFlag (set by client when pulling messages) + */ + public static final String PROPERTY_TRANSIENT_GROUP_CONFIG = PROPERTY_TRANSIENT_PREFIX + "GROUP_SYS_FLAG"; + public static final String KEY_SEPARATOR = " "; public static final HashSet STRING_HASH_SET = new HashSet<>(64); @@ -89,7 +109,11 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES); STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); STRING_HASH_SET.add(PROPERTY_POP_CK); + STRING_HASH_SET.add(PROPERTY_POP_CK_OFFSET); STRING_HASH_SET.add(PROPERTY_FIRST_POP_TIME); + STRING_HASH_SET.add(PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); + STRING_HASH_SET.add(DUP_INFO); + STRING_HASH_SET.add(PROPERTY_EXTEND_UNIQ_INFO); STRING_HASH_SET.add(PROPERTY_INSTANCE_ID); STRING_HASH_SET.add(PROPERTY_CORRELATION_ID); STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO_CLIENT); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java index 813ad891f6..d6d4a8f4fd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageDecoder.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common.message; +import java.io.IOException; import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -263,13 +264,132 @@ public class MessageDecoder { return byteBuffer.array(); } + /** + * Encode without store timestamp and store host, skip blank msg. + * + * @param messageExt msg + * @param needCompress need compress or not + * @return byte array + * @throws IOException when compress failed + */ + public static byte[] encodeUniquely(MessageExt messageExt, boolean needCompress) throws IOException { + byte[] body = messageExt.getBody(); + byte[] topics = messageExt.getTopic().getBytes(CHARSET_UTF8); + byte topicLen = (byte) topics.length; + String properties = messageProperties2String(messageExt.getProperties()); + byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); + short propertiesLength = (short) propertiesBytes.length; + int sysFlag = messageExt.getSysFlag(); + int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; + byte[] newBody = messageExt.getBody(); + if (needCompress && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + newBody = UtilAll.compress(body, 5); + } + int bodyLength = newBody.length; + int storeSize = messageExt.getStoreSize(); + ByteBuffer byteBuffer; + if (storeSize > 0) { + byteBuffer = ByteBuffer.allocate(storeSize - 8); // except size for store timestamp + } else { + storeSize = 4 + // 1 TOTALSIZE + 4 + // 2 MAGICCODE + 4 + // 3 BODYCRC + 4 + // 4 QUEUEID + 4 + // 5 FLAG + 8 + // 6 QUEUEOFFSET + 8 + // 7 PHYSICALOFFSET + 4 + // 8 SYSFLAG + 8 + // 9 BORNTIMESTAMP + bornhostLength + // 10 BORNHOST + 4 + // 11 RECONSUMETIMES + 8 + // 12 Prepared Transaction Offset + 4 + bodyLength + // 13 BODY + +1 + topicLen + // 14 TOPIC + 2 + propertiesLength // 15 propertiesLength + ; + byteBuffer = ByteBuffer.allocate(storeSize); + } + + // 1 TOTALSIZE + byteBuffer.putInt(storeSize); + + // 2 MAGICCODE + byteBuffer.putInt(MESSAGE_MAGIC_CODE); + + // 3 BODYCRC + int bodyCRC = messageExt.getBodyCRC(); + byteBuffer.putInt(bodyCRC); + + // 4 QUEUEID + int queueId = messageExt.getQueueId(); + byteBuffer.putInt(queueId); + + // 5 FLAG + int flag = messageExt.getFlag(); + byteBuffer.putInt(flag); + + // 6 QUEUEOFFSET + long queueOffset = messageExt.getQueueOffset(); + byteBuffer.putLong(queueOffset); + + // 7 PHYSICALOFFSET + long physicOffset = messageExt.getCommitLogOffset(); + byteBuffer.putLong(physicOffset); + + // 8 SYSFLAG + byteBuffer.putInt(sysFlag); + + // 9 BORNTIMESTAMP + long bornTimeStamp = messageExt.getBornTimestamp(); + byteBuffer.putLong(bornTimeStamp); + + // 10 BORNHOST + InetSocketAddress bornHost = (InetSocketAddress) messageExt.getBornHost(); + byteBuffer.put(bornHost.getAddress().getAddress()); + byteBuffer.putInt(bornHost.getPort()); + + // 11 RECONSUMETIMES + int reconsumeTimes = messageExt.getReconsumeTimes(); + byteBuffer.putInt(reconsumeTimes); + + // 12 Prepared Transaction Offset + long preparedTransactionOffset = messageExt.getPreparedTransactionOffset(); + byteBuffer.putLong(preparedTransactionOffset); + + // 13 BODY + byteBuffer.putInt(bodyLength); + byteBuffer.put(newBody); + + // 14 TOPIC + byteBuffer.put(topicLen); + byteBuffer.put(topics); + + // 15 properties + byteBuffer.putShort(propertiesLength); + byteBuffer.put(propertiesBytes); + + return byteBuffer.array(); + } + public static MessageExt decode( ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody) { return decode(byteBuffer, readBody, deCompressBody, false); } public static MessageExt decode( - ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) { + java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient) { + return decode(byteBuffer, readBody, deCompressBody, isClient, false, false); + } + + public static MessageExt decode( + java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient, + final boolean isSetPropertiesString) { + return decode(byteBuffer, readBody, deCompressBody, isClient, isSetPropertiesString, false); + } + + public static MessageExt decode( + java.nio.ByteBuffer byteBuffer, final boolean readBody, final boolean deCompressBody, final boolean isClient, + final boolean isSetPropertiesString, final boolean checkCRC) { try { MessageExt msgExt; @@ -284,7 +404,10 @@ public class MessageDecoder { msgExt.setStoreSize(storeSize); // 2 MAGICCODE - byteBuffer.getInt(); + int magicCode = byteBuffer.getInt(); + if (magicCode != MESSAGE_MAGIC_CODE) { + throw new RuntimeException("Unknown magicCode: " + magicCode); + } // 3 BODYCRC int bodyCRC = byteBuffer.getInt(); @@ -347,6 +470,14 @@ public class MessageDecoder { byte[] body = new byte[bodyLen]; byteBuffer.get(body); + if (checkCRC) { + //crc body + int crc = UtilAll.crc32(body, 0, bodyLen); + if (crc != bodyCRC) { + throw new Exception("Msg crc is error!"); + } + } + // uncompress body if (deCompressBody && (sysFlag & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { body = UtilAll.uncompress(body); @@ -370,8 +501,14 @@ public class MessageDecoder { byte[] properties = new byte[propertiesLength]; byteBuffer.get(properties); String propertiesString = new String(properties, CHARSET_UTF8); - Map map = string2messageProperties(propertiesString); - msgExt.setProperties(map); + if (!isSetPropertiesString) { + Map map = string2messageProperties(propertiesString); + msgExt.setProperties(map); + } else { + Map map = string2messageProperties(propertiesString); + map.put("propertiesString", propertiesString); + msgExt.setProperties(map); + } } int msgIDLength = storehostIPLength + 4 + 8; @@ -396,9 +533,9 @@ public class MessageDecoder { } public static List decodesBatch(ByteBuffer byteBuffer, - final boolean readBody, - final boolean decompressBody, - final boolean isClient) { + final boolean readBody, + final boolean decompressBody, + final boolean isClient) { List msgExts = new ArrayList(); while (byteBuffer.hasRemaining()) { MessageExt msgExt = decode(byteBuffer, readBody, decompressBody, isClient); @@ -494,10 +631,7 @@ public class MessageDecoder { String properties = messageProperties2String(message.getProperties()); byte[] propertiesBytes = properties.getBytes(CHARSET_UTF8); //note properties length must not more than Short.MAX - int propsLen = propertiesBytes.length; - if (propsLen > Short.MAX_VALUE) - throw new RuntimeException(String.format("Properties size of message exceeded, properties size: {}, maxSize: {}.", propsLen, Short.MAX_VALUE)); - short propertiesLength = (short) propsLen; + short propertiesLength = (short) propertiesBytes.length; int sysFlag = message.getFlag(); int storeSize = 4 // 1 TOTALSIZE + 4 // 2 MAGICCOD diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java index 133cb93ba0..5905d28f41 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExt.java @@ -249,6 +249,61 @@ public class MessageExt extends Message { this.preparedTransactionOffset = preparedTransactionOffset; } + /** + * + * achieves topicSysFlag value from transient properties + * + * @return + */ + public Integer getTopicSysFlag() { + String topicSysFlagString = getProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG); + if (topicSysFlagString != null && topicSysFlagString.length() > 0) { + return Integer.valueOf(topicSysFlagString); + } + return null; + } + + /** + * set topicSysFlag to transient properties, or clear it + * + * @param topicSysFlag + */ + public void setTopicSysFlag(Integer topicSysFlag) { + if (topicSysFlag == null) { + clearProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG); + } else { + putProperty(MessageConst.PROPERTY_TRANSIENT_TOPIC_CONFIG, String.valueOf(topicSysFlag)); + } + } + + /** + * + * achieves groupSysFlag value from transient properties + * + * @return + */ + public Integer getGroupSysFlag() { + String groupSysFlagString = getProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG); + if (groupSysFlagString != null && groupSysFlagString.length() > 0) { + return Integer.valueOf(groupSysFlagString); + } + return null; + } + + /** + * + * set groupSysFlag to transient properties, or clear it + * + * @param groupSysFlag + */ + public void setGroupSysFlag(Integer groupSysFlag) { + if (groupSysFlag == null) { + clearProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG); + } else { + putProperty(MessageConst.PROPERTY_TRANSIENT_GROUP_CONFIG, String.valueOf(groupSysFlag)); + } + } + @Override public String toString() { return "MessageExt [brokerName=" + brokerName + ", queueId=" + queueId + ", storeSize=" + storeSize + ", queueOffset=" + queueOffset diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBatch.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBatch.java new file mode 100644 index 0000000000..7cdd84d2f9 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBatch.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.message; + +import java.nio.ByteBuffer; + +public class MessageExtBatch extends MessageExtBrokerInner { + + private static final long serialVersionUID = -2353110995348498537L; + /** + * Inner batch means the batch dose not need to be unwrapped + */ + private boolean isInnerBatch = false; + public ByteBuffer wrap() { + assert getBody() != null; + return ByteBuffer.wrap(getBody(), 0, getBody().length); + } + + public boolean isInnerBatch() { + return isInnerBatch; + } + + public void setInnerBatch(boolean innerBatch) { + isInnerBatch = innerBatch; + } + + private ByteBuffer encodedBuff; + + public ByteBuffer getEncodedBuff() { + return encodedBuff; + } + + public void setEncodedBuff(ByteBuffer encodedBuff) { + this.encodedBuff = encodedBuff; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java new file mode 100644 index 0000000000..7af7014f75 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.message; + +import java.nio.ByteBuffer; + +import org.apache.rocketmq.common.TopicFilterType; + +public class MessageExtBrokerInner extends MessageExt { + private static final long serialVersionUID = 7256001576878700634L; + private String propertiesString; + private long tagsCode; + + private ByteBuffer encodedBuff; + + public ByteBuffer getEncodedBuff() { + return encodedBuff; + } + + public void setEncodedBuff(ByteBuffer encodedBuff) { + this.encodedBuff = encodedBuff; + } + + public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { + if (null == tags || tags.length() == 0) { return 0; } + + return tags.hashCode(); + } + + public static long tagsString2tagsCode(final String tags) { + return tagsString2tagsCode(null, tags); + } + + public String getPropertiesString() { + return propertiesString; + } + + public void setPropertiesString(String propertiesString) { + this.propertiesString = propertiesString; + } + + public long getTagsCode() { + return tagsCode; + } + + public void setTagsCode(long tagsCode) { + this.tagsCode = tagsCode; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageType.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageType.java index 08091f47f1..3afbb1ffa1 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageType.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageType.java @@ -18,8 +18,28 @@ package org.apache.rocketmq.common.message; public enum MessageType { - Normal_Msg, - Trans_Msg_Half, - Trans_msg_Commit, - Delay_Msg, + Normal_Msg("Normal"), + Order_Msg("Order"), + Trans_Msg_Half("Trans"), + Trans_msg_Commit("TransCommit"), + Delay_Msg("Delay"); + + private final String shortName; + + MessageType(String shortName) { + this.shortName = shortName; + } + + public String getShortName() { + return shortName; + } + + public static MessageType getByShortName(String shortName) { + for (MessageType msgType : MessageType.values()) { + if (msgType.getShortName().equals(shortName)) { + return msgType; + } + } + return Normal_Msg; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageVersion.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageVersion.java new file mode 100644 index 0000000000..c81275a203 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageVersion.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.message; + +import java.nio.ByteBuffer; + +public enum MessageVersion { + + MESSAGE_VERSION_V1(MessageDecoder.MESSAGE_MAGIC_CODE) { + @Override + public int getTopicLengthSize() { + return 1; + } + + @Override + public int getTopicLength(ByteBuffer buffer) { + return buffer.get(); + } + + @Override + public int getTopicLength(ByteBuffer buffer, int index) { + return buffer.get(index); + } + + @Override + public void putTopicLength(ByteBuffer buffer, int topicLength) { + buffer.put((byte) topicLength); + } + }; + + private final int magicCode; + + MessageVersion(int magicCode) { + this.magicCode = magicCode; + } + + public static MessageVersion valueOfMagicCode(int magicCode) { + for (MessageVersion version : MessageVersion.values()) { + if (version.getMagicCode() == magicCode) { + return version; + } + } + + throw new IllegalArgumentException("Invalid magicCode " + magicCode); + } + + public int getMagicCode() { + return magicCode; + } + + public abstract int getTopicLengthSize(); + + public abstract int getTopicLength(java.nio.ByteBuffer buffer); + public abstract int getTopicLength(java.nio.ByteBuffer buffer, int index); + public abstract void putTopicLength(java.nio.ByteBuffer buffer, int topicLength); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java new file mode 100644 index 0000000000..6e7a97d47d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/DefaultTopAddressing.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.namesrv; + +import com.google.common.base.Strings; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.ServiceLoader; +import java.util.Map; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.utils.HttpTinyClient; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class DefaultTopAddressing implements TopAddressing { + + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + private String nsAddr; + private String wsAddr; + private String unitName; + private Map para; + private List topAddressingList; + + public DefaultTopAddressing(final String wsAddr) { + this(wsAddr, null); + } + + public DefaultTopAddressing(final String wsAddr, final String unitName) { + this.wsAddr = wsAddr; + this.unitName = unitName; + this.topAddressingList = loadCustomTopAddressing(); + } + + + public DefaultTopAddressing(final String unitName, final Map para, final String wsAddr) { + this.wsAddr = wsAddr; + this.unitName = unitName; + this.para = para; + this.topAddressingList = loadCustomTopAddressing(); + } + + private static String clearNewLine(final String str) { + String newString = str.trim(); + int index = newString.indexOf("\r"); + if (index != -1) { + return newString.substring(0, index); + } + + index = newString.indexOf("\n"); + if (index != -1) { + return newString.substring(0, index); + } + + return newString; + } + + private List loadCustomTopAddressing() { + ServiceLoader serviceLoader = ServiceLoader.load(TopAddressing.class); + Iterator iterator = serviceLoader.iterator(); + List topAddressingList = new ArrayList(); + if (iterator.hasNext()) { + topAddressingList.add(iterator.next()); + } + return topAddressingList; + } + + @Override + public final String fetchNSAddr() { + if (!topAddressingList.isEmpty()) { + for (TopAddressing topAddressing : topAddressingList) { + String nsAddress = topAddressing.fetchNSAddr(); + if (!Strings.isNullOrEmpty(nsAddress)) { + return nsAddress; + } + } + } + // Return result of default implementation + return fetchNSAddr(true, 3000); + } + + @Override + public void registerChangeCallBack(NameServerUpdateCallback changeCallBack) { + if (!topAddressingList.isEmpty()) { + for (TopAddressing topAddressing : topAddressingList) { + topAddressing.registerChangeCallBack(changeCallBack); + } + } + } + + public final String fetchNSAddr(boolean verbose, long timeoutMills) { + String url = this.wsAddr; + try { + if (null != para && para.size() > 0) { + if (!UtilAll.isBlank(this.unitName)) { + url = url + "-" + this.unitName + "?nofix=1&"; + } + else { + url = url + "?"; + } + for (Map.Entry entry : this.para.entrySet()) { + url += entry.getKey() + "=" + entry.getValue() + "&"; + } + url = url.substring(0, url.length() - 1); + } + else { + if (!UtilAll.isBlank(this.unitName)) { + url = url + "-" + this.unitName + "?nofix=1"; + } + } + + HttpTinyClient.HttpResult result = HttpTinyClient.httpGet(url, null, null, "UTF-8", timeoutMills); + if (200 == result.code) { + String responseStr = result.content; + if (responseStr != null) { + return clearNewLine(responseStr); + } else { + LOGGER.error("fetch nameserver address is null"); + } + } else { + LOGGER.error("fetch nameserver address failed. statusCode=" + result.code); + } + } catch (IOException e) { + if (verbose) { + LOGGER.error("fetch name server address exception", e); + } + } + + if (verbose) { + String errorMsg = + "connect to " + url + " failed, maybe the domain name " + MixAll.getWSAddr() + " not bind in /etc/hosts"; + errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL); + + LOGGER.warn(errorMsg); + } + return null; + } + + public String getNsAddr() { + return nsAddr; + } + + public void setNsAddr(String nsAddr) { + this.nsAddr = nsAddr; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/NameServerUpdateCallback.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/NameServerUpdateCallback.java new file mode 100644 index 0000000000..67ce2b8682 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/NameServerUpdateCallback.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.namesrv; + +public interface NameServerUpdateCallback { + String onNameServerAddressChange(String namesrvAddress); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java index f687d2c243..002c9569f6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/NamesrvConfig.java @@ -35,6 +35,51 @@ public class NamesrvConfig { private String productEnvName = "center"; private boolean clusterTest = false; private boolean orderMessageEnable = false; + private boolean returnOrderTopicConfigToBroker = true; + private volatile boolean remoteFaultTolerance = true; + + /** + * Indicates the nums of thread to handle client requests, like GET_ROUTEINTO_BY_TOPIC. + */ + private int clientRequestThreadPoolNums = 8; + /** + * Indicates the nums of thread to handle broker or operation requests, like REGISTER_BROKER. + */ + private int defaultThreadPoolNums = 16; + /** + * Indicates the capacity of queue to hold client requests. + */ + private int clientRequestThreadPoolQueueCapacity = 50000; + /** + * Indicates the capacity of queue to hold broker or operation requests. + */ + private int defaultThreadPoolQueueCapacity = 10000; + /** + * Interval of periodic scanning for non-active broker; + */ + private long scanNotActiveBrokerInterval = 5 * 1000; + + private int unRegisterBrokerQueueCapacity = 3000; + + /** + * Support acting master or not. + * + * The slave can be an acting master when master node is down to support following operations: + * 1. support lock/unlock message queue operation. + * 2. support searchOffset, query maxOffset/minOffset operation. + * 3. support query earliest msg store time. + */ + private boolean supportActingMaster = false; + + private volatile boolean enableAllTopicList = false; + + public void setRemoteFaultTolerance(boolean remoteFaultTolerance) { + this.remoteFaultTolerance = remoteFaultTolerance; + } + + public boolean isRemoteFaultTolerance() { + return remoteFaultTolerance; + } public boolean isOrderMessageEnable() { return orderMessageEnable; @@ -83,4 +128,76 @@ public class NamesrvConfig { public void setConfigStorePath(final String configStorePath) { this.configStorePath = configStorePath; } + + public boolean isReturnOrderTopicConfigToBroker() { + return returnOrderTopicConfigToBroker; + } + + public void setReturnOrderTopicConfigToBroker(boolean returnOrderTopicConfigToBroker) { + this.returnOrderTopicConfigToBroker = returnOrderTopicConfigToBroker; + } + + public int getClientRequestThreadPoolNums() { + return clientRequestThreadPoolNums; + } + + public void setClientRequestThreadPoolNums(final int clientRequestThreadPoolNums) { + this.clientRequestThreadPoolNums = clientRequestThreadPoolNums; + } + + public int getDefaultThreadPoolNums() { + return defaultThreadPoolNums; + } + + public void setDefaultThreadPoolNums(final int defaultThreadPoolNums) { + this.defaultThreadPoolNums = defaultThreadPoolNums; + } + + public int getClientRequestThreadPoolQueueCapacity() { + return clientRequestThreadPoolQueueCapacity; + } + + public void setClientRequestThreadPoolQueueCapacity(final int clientRequestThreadPoolQueueCapacity) { + this.clientRequestThreadPoolQueueCapacity = clientRequestThreadPoolQueueCapacity; + } + + public int getDefaultThreadPoolQueueCapacity() { + return defaultThreadPoolQueueCapacity; + } + + public void setDefaultThreadPoolQueueCapacity(final int defaultThreadPoolQueueCapacity) { + this.defaultThreadPoolQueueCapacity = defaultThreadPoolQueueCapacity; + } + + public long getScanNotActiveBrokerInterval() { + return scanNotActiveBrokerInterval; + } + + public void setScanNotActiveBrokerInterval(long scanNotActiveBrokerInterval) { + this.scanNotActiveBrokerInterval = scanNotActiveBrokerInterval; + } + + public int getUnRegisterBrokerQueueCapacity() { + return unRegisterBrokerQueueCapacity; + } + + public void setUnRegisterBrokerQueueCapacity(final int unRegisterBrokerQueueCapacity) { + this.unRegisterBrokerQueueCapacity = unRegisterBrokerQueueCapacity; + } + + public boolean isSupportActingMaster() { + return supportActingMaster; + } + + public void setSupportActingMaster(final boolean supportActingMaster) { + this.supportActingMaster = supportActingMaster; + } + + public boolean isEnableAllTopicList() { + return enableAllTopicList; + } + + public void setEnableAllTopicList(boolean enableAllTopicList) { + this.enableAllTopicList = enableAllTopicList; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java index 88a4122b6d..3ca182825b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java +++ b/common/src/main/java/org/apache/rocketmq/common/namesrv/TopAddressing.java @@ -14,94 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -/** - * $Id: TopAddressing.java 1831 2013-05-16 01:39:51Z vintagewang@apache.org $ - */ package org.apache.rocketmq.common.namesrv; -import java.io.IOException; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.utils.HttpTinyClient; -public class TopAddressing { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); +public interface TopAddressing { - private String nsAddr; - private String wsAddr; - private String unitName; + String fetchNSAddr(); - public TopAddressing(final String wsAddr) { - this(wsAddr, null); - } - - public TopAddressing(final String wsAddr, final String unitName) { - this.wsAddr = wsAddr; - this.unitName = unitName; - } - - private static String clearNewLine(final String str) { - String newString = str.trim(); - int index = newString.indexOf("\r"); - if (index != -1) { - return newString.substring(0, index); - } - - index = newString.indexOf("\n"); - if (index != -1) { - return newString.substring(0, index); - } - - return newString; - } - - public final String fetchNSAddr() { - return fetchNSAddr(true, 3000); - } - - public final String fetchNSAddr(boolean verbose, long timeoutMills) { - String url = this.wsAddr; - try { - if (!UtilAll.isBlank(this.unitName)) { - url = url + "-" + this.unitName + "?nofix=1"; - } - HttpTinyClient.HttpResult result = HttpTinyClient.httpGet(url, null, null, "UTF-8", timeoutMills); - if (200 == result.code) { - String responseStr = result.content; - if (responseStr != null) { - return clearNewLine(responseStr); - } else { - log.error("fetch nameserver address is null"); - } - } else { - log.error("fetch nameserver address failed. statusCode=" + result.code); - } - } catch (IOException e) { - if (verbose) { - log.error("fetch name server address exception", e); - } - } - - if (verbose) { - String errorMsg = - "connect to " + url + " failed, maybe the domain name " + MixAll.getWSAddr() + " not bind in /etc/hosts"; - errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL); - - log.warn(errorMsg); - } - return null; - } - - public String getNsAddr() { - return nsAddr; - } - - public void setNsAddr(String nsAddr) { - this.nsAddr = nsAddr; - } + void registerChangeCallBack(NameServerUpdateCallback changeCallBack); } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/ForbiddenType.java b/common/src/main/java/org/apache/rocketmq/common/protocol/ForbiddenType.java new file mode 100644 index 0000000000..1c661c2a41 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/ForbiddenType.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol; + +/** + * + * gives the reason for a no permission messaging pulling. + * + */ +public interface ForbiddenType { + + /** + * 1=forbidden by broker + */ + int BROKER_FORBIDDEN = 1; + /** + * 2=forbidden by groupId + */ + int GROUP_FORBIDDEN = 2; + /** + * 3=forbidden by topic + */ + int TOPIC_FORBIDDEN = 3; + /** + * 4=forbidden by brocasting mode + */ + int BROADCASTING_DISABLE_FORBIDDEN = 4; + /** + * 5=forbidden for a substription(group with a topic) + */ + int SUBSCRIPTION_FORBIDDEN = 5; + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 534a2d25ec..541227f4aa 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -70,6 +70,8 @@ public class RequestCode { public static final int CHECK_CLIENT_CONFIG = 46; + public static final int GET_CLIENT_CONFIG = 47; + public static final int UPDATE_AND_CREATE_ACL_CONFIG = 50; public static final int DELETE_ACL_CONFIG = 51; @@ -82,7 +84,10 @@ public class RequestCode { public static final int POP_MESSAGE = 200050; public static final int ACK_MESSAGE = 200051; + public static final int PEEK_MESSAGE = 200052; public static final int CHANGE_MESSAGE_INVISIBLETIME = 200053; + public static final int NOTIFICATION = 200054; + public static final int POLLING_INFO = 200055; public static final int PUT_KV_CONFIG = 100; @@ -121,6 +126,7 @@ public class RequestCode { public static final int DELETE_TOPIC_IN_BROKER = 215; public static final int DELETE_TOPIC_IN_NAMESRV = 216; + public static final int REGISTER_TOPIC_IN_NAMESRV = 217; public static final int GET_KVLIST_BY_NAMESPACE = 219; public static final int RESET_CONSUMER_CLIENT_OFFSET = 220; @@ -135,6 +141,9 @@ public class RequestCode { public static final int GET_TOPICS_BY_CLUSTER = 224; + public static final int QUERY_TOPICS_BY_CONSUMER = 343; + public static final int QUERY_SUBSCRIPTION_BY_CONSUMER = 345; + public static final int REGISTER_FILTER_SERVER = 301; public static final int REGISTER_MESSAGE_FILTER_CLASS = 302; @@ -195,11 +204,36 @@ public class RequestCode { public static final int ADD_WRITE_PERM_OF_BROKER = 327; + public static final int GET_TOPIC_CONFIG = 351; + public static final int GET_SUBSCRIPTIONGROUP_CONFIG = 352; + public static final int UPDATE_AND_GET_GROUP_FORBIDDEN = 353; + + public static final int LITE_PULL_MESSAGE = 361; + public static final int QUERY_ASSIGNMENT = 400; public static final int SET_MESSAGE_REQUEST_MODE = 401; public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513; + /** + * Below request codes are used by broker container, + * these request codes are started with '9'. + */ + public static final int GET_BROKER_MEMBER_GROUP = 901; + + public static final int ADD_BROKER = 902; + + public static final int REMOVE_BROKER = 903; + + public static final int BROKER_HEARTBEAT = 904; + + public static final int NOTIFY_MIN_BROKER_ID_CHANGE = 905; + + public static final int EXCHANGE_BROKER_HA_INFO = 906; + + public static final int GET_BROKER_HA_STATUS = 907; + + public static final int RESET_MASTER_FLUSH_OFFSET = 908; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java index 3f691b5414..db2a275e35 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java @@ -20,7 +20,6 @@ package org.apache.rocketmq.common.protocol; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; public class ResponseCode extends RemotingSysResponseCode { - public static final int ASYNC_AND_RETURN_NULL = -2; public static final int FLUSH_DISK_TIMEOUT = 10; @@ -85,6 +84,12 @@ public class ResponseCode extends RemotingSysResponseCode { public static final int POLLING_TIMEOUT = 210; + public static final int BROKER_NOT_EXIST = 211; + + public static final int BROKER_DISPATCH_NOT_COMPLETE = 212; + + public static final int FLOW_CONTROL = 215; + public static final int NOT_LEADER_FOR_QUEUE = 501; public static final int RPC_UNKNOWN = -1000; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/BrokerMemberGroup.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/BrokerMemberGroup.java new file mode 100644 index 0000000000..eb02f88678 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/BrokerMemberGroup.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import com.google.common.base.Objects; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class BrokerMemberGroup { + private String cluster; + private String brokerName; + private Map brokerAddrs; + + // Provide default constructor for serializer + public BrokerMemberGroup() { + this.brokerAddrs = new HashMap(); + } + + public BrokerMemberGroup(final String cluster, final String brokerName) { + this.cluster = cluster; + this.brokerName = brokerName; + this.brokerAddrs = new HashMap(); + } + + public long minimumBrokerId() { + if (this.brokerAddrs.isEmpty()) { + return 0; + } + return Collections.min(brokerAddrs.keySet()); + } + + public String getCluster() { + return cluster; + } + + public void setCluster(final String cluster) { + this.cluster = cluster; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(final String brokerName) { + this.brokerName = brokerName; + } + + public Map getBrokerAddrs() { + return brokerAddrs; + } + + public void setBrokerAddrs(final Map brokerAddrs) { + this.brokerAddrs = brokerAddrs; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BrokerMemberGroup that = (BrokerMemberGroup) o; + return Objects.equal(cluster, that.cluster) && + Objects.equal(brokerName, that.brokerName) && + Objects.equal(brokerAddrs, that.brokerAddrs); + } + + @Override + public int hashCode() { + return Objects.hashCode(cluster, brokerName, brokerAddrs); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java index a78ce55456..70d6011006 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CheckClientRequestBody.java @@ -25,6 +25,7 @@ public class CheckClientRequestBody extends RemotingSerializable { private String clientId; private String group; private SubscriptionData subscriptionData; + private String namespace; public String getClientId() { return clientId; @@ -49,4 +50,12 @@ public class CheckClientRequestBody extends RemotingSerializable { public void setSubscriptionData(SubscriptionData subscriptionData) { this.subscriptionData = subscriptionData; } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java index 222e51a3a6..700cff50cc 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ClusterInfo.java @@ -19,29 +19,29 @@ package org.apache.rocketmq.common.protocol.body; import com.google.common.base.Objects; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Map; import java.util.List; import java.util.Set; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class ClusterInfo extends RemotingSerializable { - private HashMap brokerAddrTable; - private HashMap> clusterAddrTable; + private Map brokerAddrTable; + private Map> clusterAddrTable; - public HashMap getBrokerAddrTable() { + public Map getBrokerAddrTable() { return brokerAddrTable; } - public void setBrokerAddrTable(HashMap brokerAddrTable) { + public void setBrokerAddrTable(Map brokerAddrTable) { this.brokerAddrTable = brokerAddrTable; } - public HashMap> getClusterAddrTable() { + public Map> getClusterAddrTable() { return clusterAddrTable; } - public void setClusterAddrTable(HashMap> clusterAddrTable) { + public void setClusterAddrTable(Map> clusterAddrTable) { this.clusterAddrTable = clusterAddrTable; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java index 5b08d788d9..1317d548cd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java @@ -19,11 +19,13 @@ package org.apache.rocketmq.common.protocol.body; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class ConsumerOffsetSerializeWrapper extends RemotingSerializable { private ConcurrentMap> offsetTable = new ConcurrentHashMap>(512); + private DataVersion dataVersion; public ConcurrentMap> getOffsetTable() { return offsetTable; @@ -32,4 +34,12 @@ public class ConsumerOffsetSerializeWrapper extends RemotingSerializable { public void setOffsetTable(ConcurrentMap> offsetTable) { this.offsetTable = offsetTable; } + + public DataVersion getDataVersion() { + return dataVersion; + } + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfo.java index 0543adde54..6b0355fd85 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfo.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ConsumerRunningInfo.java @@ -45,6 +45,8 @@ public class ConsumerRunningInfo extends RemotingSerializable { private TreeMap statusTable = new TreeMap(); + private TreeMap userConsumerInfo = new TreeMap(); + private String jstack; public static boolean analyzeSubscription(final TreeMap criTable) { @@ -87,13 +89,14 @@ public class ConsumerRunningInfo extends RemotingSerializable { prev = next.getValue(); } - if (prev != null) { - - if (prev.getSubscriptionSet().isEmpty()) { - // Subscription empty! - return false; - } - } + // after consumer.unsubscribe , SubscriptionSet is Empty + //if (prev != null) { + // + // if (prev.getSubscriptionSet().isEmpty()) { + // // Subscription empty! + // return false; + // } + //} } } @@ -193,6 +196,10 @@ public class ConsumerRunningInfo extends RemotingSerializable { this.statusTable = statusTable; } + public TreeMap getUserConsumerInfo() { + return userConsumerInfo; + } + public String formatString() { StringBuilder sb = new StringBuilder(); @@ -318,6 +325,16 @@ public class ConsumerRunningInfo extends RemotingSerializable { } } + if (this.userConsumerInfo != null) { + sb.append("\n\n#User Consume Info#\n"); + Iterator> it = this.userConsumerInfo.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-40s: %s%n", next.getKey(), next.getValue()); + sb.append(item); + } + } + if (this.jstack != null) { sb.append("\n\n#Consumer jstack#\n"); sb.append(this.jstack); diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetBrokerMemberGroupResponseBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetBrokerMemberGroupResponseBody.java new file mode 100644 index 0000000000..27625877c2 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetBrokerMemberGroupResponseBody.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class GetBrokerMemberGroupResponseBody extends RemotingSerializable { + // Contains the broker member info of the same broker group + private BrokerMemberGroup brokerMemberGroup; + + public BrokerMemberGroup getBrokerMemberGroup() { + return brokerMemberGroup; + } + + public void setBrokerMemberGroup(final BrokerMemberGroup brokerMemberGroup) { + this.brokerMemberGroup = brokerMemberGroup; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetRemoteClientConfigBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetRemoteClientConfigBody.java new file mode 100644 index 0000000000..38b3b482a8 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/GetRemoteClientConfigBody.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.List; + +public class GetRemoteClientConfigBody extends RemotingSerializable { + private List keys = new ArrayList(); + + public List getKeys() { + return keys; + } + + public void setKeys(List keys) { + this.keys = keys; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/HARuntimeInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/HARuntimeInfo.java new file mode 100644 index 0000000000..8994b51633 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/HARuntimeInfo.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class HARuntimeInfo extends RemotingSerializable { + + private boolean master; + private long masterCommitLogMaxOffset; + private int inSyncSlaveNums; + private List haConnectionInfo = new ArrayList(); + private HAClientRuntimeInfo haClientRuntimeInfo = new HAClientRuntimeInfo(); + + public boolean isMaster() { + return this.master; + } + + public void setMaster(boolean master) { + this.master = master; + } + + public long getMasterCommitLogMaxOffset() { + return this.masterCommitLogMaxOffset; + } + + public void setMasterCommitLogMaxOffset(long masterCommitLogMaxOffset) { + this.masterCommitLogMaxOffset = masterCommitLogMaxOffset; + } + + public int getInSyncSlaveNums() { + return this.inSyncSlaveNums; + } + + public void setInSyncSlaveNums(int inSyncSlaveNums) { + this.inSyncSlaveNums = inSyncSlaveNums; + } + + public List getHaConnectionInfo() { + return this.haConnectionInfo; + } + + public void setHaConnectionInfo(List haConnectionInfo) { + this.haConnectionInfo = haConnectionInfo; + } + + public HAClientRuntimeInfo getHaClientRuntimeInfo() { + return this.haClientRuntimeInfo; + } + + public void setHaClientRuntimeInfo(HAClientRuntimeInfo haClientRuntimeInfo) { + this.haClientRuntimeInfo = haClientRuntimeInfo; + } + + public static class HAConnectionRuntimeInfo extends RemotingSerializable { + private String addr; + private long slaveAckOffset; + private long diff; + private boolean inSync; + private long transferredByteInSecond; + private long transferFromWhere; + + public String getAddr() { + return this.addr; + } + + public void setAddr(String addr) { + this.addr = addr; + } + + public long getSlaveAckOffset() { + return this.slaveAckOffset; + } + + public void setSlaveAckOffset(long slaveAckOffset) { + this.slaveAckOffset = slaveAckOffset; + } + + public long getDiff() { + return this.diff; + } + + public void setDiff(long diff) { + this.diff = diff; + } + + public boolean isInSync() { + return this.inSync; + } + + public void setInSync(boolean inSync) { + this.inSync = inSync; + } + + public long getTransferredByteInSecond() { + return this.transferredByteInSecond; + } + + public void setTransferredByteInSecond(long transferredByteInSecond) { + this.transferredByteInSecond = transferredByteInSecond; + } + + public long getTransferFromWhere() { + return transferFromWhere; + } + + public void setTransferFromWhere(long transferFromWhere) { + this.transferFromWhere = transferFromWhere; + } + } + + public static class HAClientRuntimeInfo extends RemotingSerializable { + private String masterAddr; + private long transferredByteInSecond; + private long maxOffset; + private long lastReadTimestamp; + private long lastWriteTimestamp; + private long masterFlushOffset; + private boolean isActivated = false; + + public String getMasterAddr() { + return this.masterAddr; + } + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } + + public long getTransferredByteInSecond() { + return this.transferredByteInSecond; + } + + public void setTransferredByteInSecond(long transferredByteInSecond) { + this.transferredByteInSecond = transferredByteInSecond; + } + + public long getMaxOffset() { + return this.maxOffset; + } + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + public long getLastReadTimestamp() { + return this.lastReadTimestamp; + } + + public void setLastReadTimestamp(long lastReadTimestamp) { + this.lastReadTimestamp = lastReadTimestamp; + } + + public long getLastWriteTimestamp() { + return this.lastWriteTimestamp; + } + + public void setLastWriteTimestamp(long lastWriteTimestamp) { + this.lastWriteTimestamp = lastWriteTimestamp; + } + + public long getMasterFlushOffset() { + return masterFlushOffset; + } + + public void setMasterFlushOffset(long masterFlushOffset) { + this.masterFlushOffset = masterFlushOffset; + } + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/LockBatchRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/LockBatchRequestBody.java index 480862b8f4..92e4d4e48d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/LockBatchRequestBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/LockBatchRequestBody.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class LockBatchRequestBody extends RemotingSerializable { private String consumerGroup; private String clientId; + private boolean onlyThisBroker = false; private Set mqSet = new HashSet(); public String getConsumerGroup() { @@ -43,6 +44,14 @@ public class LockBatchRequestBody extends RemotingSerializable { this.clientId = clientId; } + public boolean isOnlyThisBroker() { + return onlyThisBroker; + } + + public void setOnlyThisBroker(boolean onlyThisBroker) { + this.onlyThisBroker = onlyThisBroker; + } + public Set getMqSet() { return mqSet; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/QuerySubscriptionResponseBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/QuerySubscriptionResponseBody.java new file mode 100644 index 0000000000..413f8c47b0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/QuerySubscriptionResponseBody.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class QuerySubscriptionResponseBody extends RemotingSerializable { + + private SubscriptionData subscriptionData; + private String group; + private String topic; + + public SubscriptionData getSubscriptionData() { + return subscriptionData; + } + + public void setSubscriptionData(SubscriptionData subscriptionData) { + this.subscriptionData = subscriptionData; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/UnlockBatchRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/UnlockBatchRequestBody.java index baf4071306..f71dafb65b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/UnlockBatchRequestBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/UnlockBatchRequestBody.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class UnlockBatchRequestBody extends RemotingSerializable { private String consumerGroup; private String clientId; + private boolean onlyThisBroker = false; private Set mqSet = new HashSet(); public String getConsumerGroup() { @@ -43,6 +44,14 @@ public class UnlockBatchRequestBody extends RemotingSerializable { this.clientId = clientId; } + public boolean isOnlyThisBroker() { + return onlyThisBroker; + } + + public void setOnlyThisBroker(boolean onlyThisBroker) { + this.onlyThisBroker = onlyThisBroker; + } + public Set getMqSet() { return mqSet; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/AddBrokerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/AddBrokerRequestHeader.java new file mode 100644 index 0000000000..e38e350425 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/AddBrokerRequestHeader.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class AddBrokerRequestHeader implements CommandCustomHeader { + @CFNullable + private String configPath; + + + @Override public void checkFields() throws RemotingCommandException { + + } + + public String getConfigPath() { + return configPath; + } + + public void setConfigPath(String configPath) { + this.configPath = configPath; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java index 1bd089d7a7..7bad63985d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java @@ -31,6 +31,12 @@ public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomH private String msgId; @CFNullable private String brokerName; + @CFNullable + private String topic; + @CFNullable + private Integer topicSysFlag; + @CFNullable + private Integer groupSysFlag; @Override public void checkFields() throws RemotingCommandException { @@ -67,4 +73,28 @@ public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomH public void setMsgId(String msgId) { this.msgId = msgId; } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public Integer getTopicSysFlag() { + return topicSysFlag; + } + + public void setTopicSysFlag(Integer topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + public Integer getGroupSysFlag() { + return groupSysFlag; + } + + public void setGroupSysFlag(Integer groupSysFlag) { + this.groupSysFlag = groupSysFlag; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java index 6591d7780c..8a33564a51 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java @@ -25,7 +25,7 @@ public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader @CFNotNull private String groupName; - private boolean removeOffset; + private boolean cleanOffset = false; @Override public void checkFields() throws RemotingCommandException { @@ -39,11 +39,11 @@ public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader this.groupName = groupName; } - public boolean isRemoveOffset() { - return removeOffset; + public boolean isCleanOffset() { + return cleanOffset; } - public void setRemoveOffset(boolean removeOffset) { - this.removeOffset = removeOffset; + public void setCleanOffset(boolean cleanOffset) { + this.cleanOffset = cleanOffset; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoRequestHeader.java new file mode 100644 index 0000000000..4a59320531 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoRequestHeader.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class ExchangeHAInfoRequestHeader implements CommandCustomHeader { + @CFNullable + public String masterHaAddress; + + @CFNullable + public Long masterFlushOffset; + + @CFNullable + public String masterAddress; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getMasterHaAddress() { + return masterHaAddress; + } + + public void setMasterHaAddress(String masterHaAddress) { + this.masterHaAddress = masterHaAddress; + } + + public Long getMasterFlushOffset() { + return masterFlushOffset; + } + + public void setMasterFlushOffset(Long masterFlushOffset) { + this.masterFlushOffset = masterFlushOffset; + } + + public String getMasterAddress() { + return masterAddress; + } + + public void setMasterAddress(String masterAddress) { + this.masterAddress = masterAddress; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoResponseHeader.java new file mode 100644 index 0000000000..7b6ea772f5 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ExchangeHAInfoResponseHeader.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class ExchangeHAInfoResponseHeader implements CommandCustomHeader { + @CFNullable + public String masterHaAddress; + + @CFNullable + public Long masterFlushOffset; + + @CFNullable + public String masterAddress; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getMasterHaAddress() { + return masterHaAddress; + } + + public void setMasterHaAddress(String masterHaAddress) { + this.masterHaAddress = masterHaAddress; + } + + public Long getMasterFlushOffset() { + return masterFlushOffset; + } + + public void setMasterFlushOffset(Long masterFlushOffset) { + this.masterFlushOffset = masterFlushOffset; + } + + public String getMasterAddress() { + return masterAddress; + } + + public void setMasterAddress(String masterAddress) { + this.masterAddress = masterAddress; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerMemberGroupRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerMemberGroupRequestHeader.java new file mode 100644 index 0000000000..c111463b82 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetBrokerMemberGroupRequestHeader.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class GetBrokerMemberGroupRequestHeader implements CommandCustomHeader { + @CFNotNull + private String clusterName; + + @CFNotNull + private String brokerName; + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(final String clusterName) { + this.clusterName = clusterName; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(final String brokerName) { + this.brokerName = brokerName; + } + + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java index e961af95e1..f58e050da7 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -22,6 +22,7 @@ package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader { @@ -30,6 +31,15 @@ public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private Integer queueId; + /** + * A message at committed offset has been dispatched from Topic to MessageQueue, so it can be consumed immediately, + * while a message at inflight offset is not visible for a consumer temporarily. + * Set this flag true if the max committed offset is needed, or false if the max inflight offset is preferred. + * The default value is true. + */ + @CFNullable + private boolean committed = true; + @Override public void checkFields() throws RemotingCommandException { } @@ -54,4 +64,11 @@ public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader { this.queueId = queueId; } + public boolean isCommitted() { + return committed; + } + + public void setCommitted(final boolean committed) { + this.committed = committed; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetSubscriptionGroupConfigRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetSubscriptionGroupConfigRequestHeader.java new file mode 100644 index 0000000000..8b0cfbf730 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetSubscriptionGroupConfigRequestHeader.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: GetAllTopicConfigResponseHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class GetSubscriptionGroupConfigRequestHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } + + @CFNotNull + private String group; + + /** + * @return the group + */ + public String getGroup() { + return group; + } + + /** + * @param group the group to set + */ + public void setGroup(String group) { + this.group = group; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/InitConsumerOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/InitConsumerOffsetRequestHeader.java new file mode 100644 index 0000000000..27a64ed1c0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/InitConsumerOffsetRequestHeader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class InitConsumerOffsetRequestHeader implements CommandCustomHeader { + + private String topic; + // @see ConsumeInitMode + private int initMode; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getInitMode() { + return initMode; + } + + public void setInitMode(int initMode) { + this.initMode = initMode; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationRequestHeader.java new file mode 100644 index 0000000000..79db24e441 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationRequestHeader.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + + +public class NotificationRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private int queueId; + @CFNotNull + private long pollTime; + @CFNotNull + private long bornTime; + + @CFNotNull + @Override + public void checkFields() throws RemotingCommandException { + } + + public long getPollTime() { + return pollTime; + } + + public void setPollTime(long pollTime) { + this.pollTime = pollTime; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public long getBornTime() { + return bornTime; + } + + public void setBornTime(long bornTime) { + this.bornTime = bornTime; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + if (queueId < 0) { + return -1; + } + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationResponseHeader.java new file mode 100644 index 0000000000..20c19d6085 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotificationResponseHeader.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class NotificationResponseHeader implements CommandCustomHeader { + + + @CFNotNull + private boolean hasMsg = false; + + public boolean isHasMsg() { + return hasMsg; + } + + public void setHasMsg(boolean hasMsg) { + this.hasMsg = hasMsg; + } + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java new file mode 100644 index 0000000000..c5ca4efa61 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/NotifyMinBrokerIdChangeRequestHeader.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class NotifyMinBrokerIdChangeRequestHeader implements CommandCustomHeader { + @CFNullable + private Long minBrokerId; + + @CFNullable + private String brokerName; + + @CFNullable + private String minBrokerAddr; + + @CFNullable + private String offlineBrokerAddr; + + @CFNullable + private String haBrokerAddr; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public Long getMinBrokerId() { + return minBrokerId; + } + + public void setMinBrokerId(Long minBrokerId) { + this.minBrokerId = minBrokerId; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + public String getMinBrokerAddr() { + return minBrokerAddr; + } + + public void setMinBrokerAddr(String minBrokerAddr) { + this.minBrokerAddr = minBrokerAddr; + } + + public String getOfflineBrokerAddr() { + return offlineBrokerAddr; + } + + public void setOfflineBrokerAddr(String offlineBrokerAddr) { + this.offlineBrokerAddr = offlineBrokerAddr; + } + + public String getHaBrokerAddr() { + return haBrokerAddr; + } + + public void setHaBrokerAddr(String haBrokerAddr) { + this.haBrokerAddr = haBrokerAddr; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PeekMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PeekMessageRequestHeader.java new file mode 100644 index 0000000000..ba172f57da --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PeekMessageRequestHeader.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class PeekMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private int queueId; + @CFNotNull + private int maxMsgNums; + @CFNotNull + private String consumerGroup; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + public int getMaxMsgNums() { + return maxMsgNums; + } + + public void setMaxMsgNums(int maxMsgNums) { + this.maxMsgNums = maxMsgNums; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoRequestHeader.java new file mode 100644 index 0000000000..a57fa6e7a8 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoRequestHeader.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + + +public class PollingInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private int queueId; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + if (queueId < 0) { + return -1; + } + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoResponseHeader.java new file mode 100644 index 0000000000..37962f9b25 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PollingInfoResponseHeader.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class PollingInfoResponseHeader implements CommandCustomHeader { + + + @CFNotNull + private int pollingNum; + + public int getPollingNum() { + return pollingNum; + } + + public void setPollingNum(int pollingNum) { + this.pollingNum = pollingNum; + } + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java index 88af984df8..84e6daf120 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java @@ -36,6 +36,12 @@ public class PullMessageResponseHeader implements CommandCustomHeader { private Long maxOffset; @CFNullable private Long offsetDelta; + @CFNullable + private Integer topicSysFlag; + @CFNullable + private Integer groupSysFlag; + @CFNullable + private Integer forbiddenType; @Override public void checkFields() throws RemotingCommandException { @@ -73,6 +79,33 @@ public class PullMessageResponseHeader implements CommandCustomHeader { this.suggestWhichBrokerId = suggestWhichBrokerId; } + public Integer getTopicSysFlag() { + return topicSysFlag; + } + + public void setTopicSysFlag(Integer topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + public Integer getGroupSysFlag() { + return groupSysFlag; + } + + public void setGroupSysFlag(Integer groupSysFlag) { + this.groupSysFlag = groupSysFlag; + } + /** + * @return the forbiddenType + */ + public Integer getForbiddenType() { + return forbiddenType; + } + /** + */ + public void setForbiddenType(Integer forbiddenType) { + this.forbiddenType = forbiddenType; + } + public Long getOffsetDelta() { return offsetDelta; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QuerySubscriptionByConsumerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QuerySubscriptionByConsumerRequestHeader.java new file mode 100644 index 0000000000..94a5ae59d6 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QuerySubscriptionByConsumerRequestHeader.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class QuerySubscriptionByConsumerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String group; + private String topic; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicsByConsumerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicsByConsumerRequestHeader.java new file mode 100644 index 0000000000..6a382d72d2 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicsByConsumerRequestHeader.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class QueryTopicsByConsumerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String group; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/RemoveBrokerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/RemoveBrokerRequestHeader.java new file mode 100644 index 0000000000..05a8f11fc1 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/RemoveBrokerRequestHeader.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class RemoveBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerClusterName; + @CFNotNull + private Long brokerId; + + @Override public void checkFields() throws RemotingCommandException { + + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + public String getBrokerClusterName() { + return brokerClusterName; + } + + public void setBrokerClusterName(String brokerClusterName) { + this.brokerClusterName = brokerClusterName; + } + + public Long getBrokerId() { + return brokerId; + } + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ResetMasterFlushOffsetHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ResetMasterFlushOffsetHeader.java new file mode 100644 index 0000000000..b51191908d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ResetMasterFlushOffsetHeader.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class ResetMasterFlushOffsetHeader implements CommandCustomHeader { + @CFNotNull + private Long masterFlushOffset; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public Long getMasterFlushOffset() { + return masterFlushOffset; + } + + public void setMasterFlushOffset(Long masterFlushOffset) { + this.masterFlushOffset = masterFlushOffset; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/StatisticsMessagesRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/StatisticsMessagesRequestHeader.java new file mode 100644 index 0000000000..e25e21a32d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/StatisticsMessagesRequestHeader.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class StatisticsMessagesRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private int queueId; + + private long fromTime; + private long toTime; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public int getQueueId() { + if (queueId < 0) { + return -1; + } + return queueId; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + public long getFromTime() { + return fromTime; + } + + public void setFromTime(long fromTime) { + this.fromTime = fromTime; + } + + public long getToTime() { + return toTime; + } + + public void setToTime(long toTime) { + this.toTime = toTime; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateGroupForbiddenRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateGroupForbiddenRequestHeader.java new file mode 100644 index 0000000000..902b4acfb5 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateGroupForbiddenRequestHeader.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * $Id: CreateTopicRequestHeader.java 1835 2013-05-16 02:00:50Z vintagewang@apache.org $ + */ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class UpdateGroupForbiddenRequestHeader implements CommandCustomHeader { + @CFNotNull + private String group; + @CFNotNull + private String topic; + + private Boolean readable; + + @Override + public void checkFields() throws RemotingCommandException { + + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public Boolean getReadable() { + return readable; + } + + public void setReadable(Boolean readable) { + this.readable = readable; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java new file mode 100644 index 0000000000..e21ef90863 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/BrokerHeartbeatRequestHeader.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.header.namesrv; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class BrokerHeartbeatRequestHeader implements CommandCustomHeader { + @CFNotNull + private String clusterName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String brokerName; + + @Override public void checkFields() throws RemotingCommandException { + + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java index ed8e7a3970..c9dd04b600 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/DeleteTopicFromNamesrvRequestHeader.java @@ -24,6 +24,8 @@ public class DeleteTopicFromNamesrvRequestHeader implements CommandCustomHeader @CFNotNull private String topic; + private String clusterName; + @Override public void checkFields() throws RemotingCommandException { } @@ -35,4 +37,12 @@ public class DeleteTopicFromNamesrvRequestHeader implements CommandCustomHeader public void setTopic(String topic) { this.topic = topic; } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java index a2806e6289..d1f5ca82d6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java @@ -25,6 +25,8 @@ import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; public class GetRouteInfoRequestHeader implements CommandCustomHeader { + public static char split = 1; + @CFNotNull private String topic; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java index 19175b04b3..b2c7884016 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java @@ -22,6 +22,7 @@ package org.apache.rocketmq.common.protocol.header.namesrv; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; public class RegisterBrokerRequestHeader implements CommandCustomHeader { @@ -35,11 +36,16 @@ public class RegisterBrokerRequestHeader implements CommandCustomHeader { private String haServerAddr; @CFNotNull private Long brokerId; + @CFNullable + private Long heartbeatTimeoutMillis; + @CFNullable + private Boolean enableActingMaster; private boolean compressed; private Integer bodyCrc32 = 0; + @Override public void checkFields() throws RemotingCommandException { } @@ -83,6 +89,14 @@ public class RegisterBrokerRequestHeader implements CommandCustomHeader { this.brokerId = brokerId; } + public Long getHeartbeatTimeoutMillis() { + return heartbeatTimeoutMillis; + } + + public void setHeartbeatTimeoutMillis(Long heartbeatTimeoutMillis) { + this.heartbeatTimeoutMillis = heartbeatTimeoutMillis; + } + public boolean isCompressed() { return compressed; } @@ -98,4 +112,12 @@ public class RegisterBrokerRequestHeader implements CommandCustomHeader { public void setBodyCrc32(Integer bodyCrc32) { this.bodyCrc32 = bodyCrc32; } + + public Boolean getEnableActingMaster() { + return enableActingMaster; + } + + public void setEnableActingMaster(Boolean enableActingMaster) { + this.enableActingMaster = enableActingMaster; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterTopicRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterTopicRequestHeader.java new file mode 100644 index 0000000000..ff19b28af4 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/RegisterTopicRequestHeader.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.protocol.header.namesrv; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class RegisterTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + @Override + public void checkFields() throws RemotingCommandException { + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java index 36599fbc87..73d725c699 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/BrokerData.java @@ -20,6 +20,7 @@ package org.apache.rocketmq.common.protocol.route; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.rocketmq.common.MixAll; @@ -30,10 +31,27 @@ public class BrokerData implements Comparable { private final Random random = new Random(); + /** + * Enable acting master or not, used for old version HA adaption, + */ + private boolean enableActingMaster = false; + public BrokerData() { } + public BrokerData(BrokerData brokerData) { + this.cluster = brokerData.cluster; + this.brokerName = brokerData.brokerName; + if (brokerData.brokerAddrs != null) { + this.brokerAddrs = new HashMap(); + for (final Map.Entry brokerEntry : brokerData.brokerAddrs.entrySet()) { + this.brokerAddrs.put(brokerEntry.getKey(), brokerEntry.getValue()); + } + } + this.enableActingMaster = brokerData.enableActingMaster; + } + public BrokerData(String cluster, String brokerName, HashMap brokerAddrs) { this.cluster = cluster; this.brokerName = brokerName; @@ -73,6 +91,14 @@ public class BrokerData implements Comparable { this.cluster = cluster; } + public boolean isEnableActingMaster() { + return enableActingMaster; + } + + public void setEnableActingMaster(boolean enableActingMaster) { + this.enableActingMaster = enableActingMaster; + } + @Override public int hashCode() { final int prime = 31; @@ -84,29 +110,33 @@ public class BrokerData implements Comparable { @Override public boolean equals(Object obj) { - if (this == obj) + if (this == obj) { return true; - if (obj == null) + } + if (obj == null) { return false; - if (getClass() != obj.getClass()) + } + if (getClass() != obj.getClass()) { return false; + } BrokerData other = (BrokerData) obj; if (brokerAddrs == null) { - if (other.brokerAddrs != null) + if (other.brokerAddrs != null) { return false; - } else if (!brokerAddrs.equals(other.brokerAddrs)) + } + } else if (!brokerAddrs.equals(other.brokerAddrs)) { return false; + } if (brokerName == null) { - if (other.brokerName != null) - return false; - } else if (!brokerName.equals(other.brokerName)) - return false; - return true; + return other.brokerName == null; + } else { + return brokerName.equals(other.brokerName); + } } @Override public String toString() { - return "BrokerData [brokerName=" + brokerName + ", brokerAddrs=" + brokerAddrs + "]"; + return "BrokerData [brokerName=" + brokerName + ", brokerAddrs=" + brokerAddrs + ", enableActingMaster=" + enableActingMaster + "]"; } @Override diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java index 2dbb2902c9..6e9e653f16 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/QueueData.java @@ -27,6 +27,19 @@ public class QueueData implements Comparable { private int perm; private int topicSysFlag; + public QueueData() { + + } + + // Deep copy QueueData + public QueueData(QueueData queueData) { + this.brokerName = queueData.brokerName; + this.readQueueNums = queueData.readQueueNums; + this.writeQueueNums = queueData.writeQueueNums; + this.perm = queueData.perm; + this.topicSysFlag = queueData.topicSysFlag; + } + public int getReadQueueNums() { return readQueueNums; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java index 3fe46309c0..0c02078eb1 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java @@ -25,6 +25,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; @@ -37,6 +39,9 @@ public class TopicRouteData extends RemotingSerializable { private Map topicQueueMappingByBroker; public TopicRouteData() { + queueDatas = new ArrayList(); + brokerDatas = new ArrayList(); + filterServerTable = new HashMap>(); } public TopicRouteData(TopicRouteData topicRouteData) { @@ -62,6 +67,54 @@ public class TopicRouteData extends RemotingSerializable { } } + public TopicRouteData cloneTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setOrderTopicConf(this.orderTopicConf); + + topicRouteData.getQueueDatas().addAll(this.queueDatas); + topicRouteData.getBrokerDatas().addAll(this.brokerDatas); + topicRouteData.getFilterServerTable().putAll(this.filterServerTable); + if (this.topicQueueMappingByBroker != null) { + Map cloneMap = new HashMap<>(this.topicQueueMappingByBroker); + topicRouteData.setTopicQueueMappingByBroker(cloneMap); + } + return topicRouteData; + } + + public TopicRouteData deepCloneTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setOrderTopicConf(this.orderTopicConf); + + for (final QueueData queueData : this.queueDatas) { + topicRouteData.getQueueDatas().add(new QueueData(queueData)); + } + + for (final BrokerData brokerData : this.brokerDatas) { + topicRouteData.getBrokerDatas().add(new BrokerData(brokerData)); + } + + for (final Map.Entry> listEntry : this.filterServerTable.entrySet()) { + topicRouteData.getFilterServerTable().put(listEntry.getKey(), + new ArrayList(listEntry.getValue())); + } + if (this.topicQueueMappingByBroker != null) { + Map cloneMap = new HashMap<>(this.topicQueueMappingByBroker.size()); + for (final Map.Entry entry : this.getTopicQueueMappingByBroker().entrySet()) { + TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(entry.getValue().getTopic(), entry.getValue().getTotalQueues(), entry.getValue().getBname(), entry.getValue().getEpoch()); + topicQueueMappingInfo.setDirty(entry.getValue().isDirty()); + topicQueueMappingInfo.setScope(entry.getValue().getScope()); + ConcurrentMap concurrentMap = new ConcurrentHashMap(entry.getValue().getCurrIdMap()); + topicQueueMappingInfo.setCurrIdMap(concurrentMap); + cloneMap.put(entry.getKey(), topicQueueMappingInfo); + } + topicRouteData.setTopicQueueMappingByBroker(cloneMap); + } + + return topicRouteData; + } + public List getQueueDatas() { return queueDatas; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDatas.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDatas.java new file mode 100644 index 0000000000..42a66b67d7 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDatas.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.route; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; +import java.util.Map; + +public class TopicRouteDatas extends RemotingSerializable { + + private Map topics = new HashMap(); + + public Map getTopics() { + return topics; + } + + public void setTopics(Map topics) { + this.topics = topics; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/FutureHolder.java b/common/src/main/java/org/apache/rocketmq/common/statistics/FutureHolder.java new file mode 100644 index 0000000000..6356101c92 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/FutureHolder.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; + +public class FutureHolder { + private ConcurrentMap> futureMap = new ConcurrentHashMap>(8); + + public void addFuture(T t, Future future) { + BlockingQueue list = futureMap.get(t); + if (list == null) { + list = new LinkedBlockingQueue(); + BlockingQueue old = futureMap.putIfAbsent(t, list); + if (old != null) { + list = old; + } + } + list.add(future); + } + + public void removeAllFuture(T t) { + cancelAll(t, false); + futureMap.remove(t); + } + + private void cancelAll(T t, boolean mayInterruptIfRunning) { + BlockingQueue list = futureMap.get(t); + if (list != null) { + for (Future future : list) { + future.cancel(mayInterruptIfRunning); + } + } + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/Interceptor.java b/common/src/main/java/org/apache/rocketmq/common/statistics/Interceptor.java new file mode 100644 index 0000000000..0af55e0591 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/Interceptor.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +/** + * interceptor + */ +public interface Interceptor { + /** + * increase multiple values + * + * @param deltas + */ + void inc(long... deltas); + + void reset(); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBrief.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBrief.java new file mode 100644 index 0000000000..970bff98a0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBrief.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import org.apache.commons.lang3.ArrayUtils; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class StatisticsBrief { + public static final int META_RANGE_INDEX = 0; + public static final int META_SLOT_NUM_INDEX = 1; + + // TopPercentile + private long[][] topPercentileMeta; + private AtomicInteger[] counts; + private AtomicLong totalCount; + + // max min avg total + private long max; + private long min; + private long total; + + public StatisticsBrief(long[][] topPercentileMeta) { + if (!isLegalMeta(topPercentileMeta)) { + throw new IllegalArgumentException("illegal topPercentileMeta"); + } + + this.topPercentileMeta = topPercentileMeta; + this.counts = new AtomicInteger[slotNum(topPercentileMeta)]; + this.totalCount = new AtomicLong(0); + reset(); + } + + public void reset() { + for (int i = 0; i < counts.length; i++) { + if (counts[i] == null) { + counts[i] = new AtomicInteger(0); + } else { + counts[i].set(0); + } + } + totalCount.set(0); + + synchronized (this) { + max = 0; + min = Long.MAX_VALUE; + total = 0; + } + } + + private static boolean isLegalMeta(long[][] meta) { + if (ArrayUtils.isEmpty(meta)) { + return false; + } + + for (long[] line : meta) { + if (ArrayUtils.isEmpty(line) || line.length != 2) { + return false; + } + } + + return true; + } + + private static int slotNum(long[][] meta) { + int ret = 1; + for (long[] line : meta) { + ret += line[META_SLOT_NUM_INDEX]; + } + return ret; + } + + public void sample(long value) { + int index = getSlotIndex(value); + counts[index].incrementAndGet(); + totalCount.incrementAndGet(); + + synchronized (this) { + max = Math.max(max, value); + min = Math.min(min, value); + total += value; + } + } + + public long tp999() { + return getTPValue(0.999f); + } + + public long getTPValue(float ratio) { + if (ratio <= 0 || ratio >= 1) { + ratio = 0.99f; + } + long count = totalCount.get(); + long excludes = (long)(count - count * ratio); + if (excludes == 0) { + return getMax(); + } + + int tmp = 0; + for (int i = counts.length - 1; i > 0; i--) { + tmp += counts[i].get(); + if (tmp > excludes) { + return Math.min(getSlotTPValue(i), getMax()); + } + } + return 0; + } + + private long getSlotTPValue(int index) { + int slotNumLeft = index; + for (int i = 0; i < topPercentileMeta.length; i++) { + int slotNum = (int)topPercentileMeta[i][META_SLOT_NUM_INDEX]; + if (slotNumLeft < slotNum) { + long metaRangeMax = topPercentileMeta[i][META_RANGE_INDEX]; + long metaRangeMin = 0; + if (i > 0) { + metaRangeMin = topPercentileMeta[i - 1][META_RANGE_INDEX]; + } + + return metaRangeMin + (metaRangeMax - metaRangeMin) / slotNum * (slotNumLeft + 1); + } else { + slotNumLeft -= slotNum; + } + } + // MAX_VALUE: the last slot + return Integer.MAX_VALUE; + } + + private int getSlotIndex(long num) { + int index = 0; + for (int i = 0; i < topPercentileMeta.length; i++) { + long rangeMax = topPercentileMeta[i][META_RANGE_INDEX]; + int slotNum = (int)topPercentileMeta[i][META_SLOT_NUM_INDEX]; + long rangeMin = (i > 0) ? topPercentileMeta[i - 1][META_RANGE_INDEX] : 0; + if (rangeMin <= num && num < rangeMax) { + index += (num - rangeMin) / ((rangeMax - rangeMin) / slotNum); + break; + } + + index += slotNum; + } + return index; + } + + /** + * Getters + * + * @return + */ + public long getMax() { + return max; + } + + public long getMin() { + return totalCount.get() > 0 ? min : 0; + } + + public long getTotal() { + return total; + } + + public long getCnt() { + return totalCount.get(); + } + + public double getAvg() { + return totalCount.get() != 0 ? ((double)total) / totalCount.get() : 0; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java new file mode 100644 index 0000000000..b0b6937844 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsBriefInterceptor.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.tuple.Pair; + +/** + * interceptor to generate statistics brief + */ +public class StatisticsBriefInterceptor implements Interceptor { + private int[] indexOfItems; + + private StatisticsBrief[] statisticsBriefs; + + public StatisticsBriefInterceptor(StatisticsItem item, Pair[] briefMetas) { + indexOfItems = new int[briefMetas.length]; + statisticsBriefs = new StatisticsBrief[briefMetas.length]; + for (int i = 0; i < briefMetas.length; i++) { + String name = briefMetas[i].getKey(); + int index = ArrayUtils.indexOf(item.getItemNames(), name); + if (index < 0) { + throw new IllegalArgumentException("illegal breifItemName: " + name); + } + indexOfItems[i] = index; + statisticsBriefs[i] = new StatisticsBrief(briefMetas[i].getValue()); + } + } + + @Override + public void inc(long... itemValues) { + for (int i = 0; i < indexOfItems.length; i++) { + int indexOfItem = indexOfItems[i]; + if (indexOfItem < itemValues.length) { + statisticsBriefs[i].sample(itemValues[indexOfItem]); + } + } + } + + @Override + public void reset() { + for (StatisticsBrief brief : statisticsBriefs) { + brief.reset(); + } + } + + public int[] getIndexOfItems() { + return indexOfItems; + } + + public void setIndexOfItems(int[] indexOfItems) { + this.indexOfItems = indexOfItems; + } + + public StatisticsBrief[] getStatisticsBriefs() { + return statisticsBriefs; + } + + public void setStatisticsBriefs(StatisticsBrief[] statisticsBriefs) { + this.statisticsBriefs = statisticsBriefs; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItem.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItem.java new file mode 100644 index 0000000000..23633dc5bf --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItem.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.security.InvalidParameterException; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.lang3.ArrayUtils; + +/** + * Statistics Item + */ +public class StatisticsItem { + private String statKind; + private String statObject; + + private String[] itemNames; + private AtomicLong[] itemAccumulates; + private AtomicLong invokeTimes; + + private Interceptor interceptor; + + /** + * last timestamp when the item was updated + */ + private AtomicLong lastTimeStamp; + + public StatisticsItem(String statKind, String statObject, String... itemNames) { + if (itemNames == null || itemNames.length <= 0) { + throw new InvalidParameterException("StatisticsItem \"itemNames\" is empty"); + } + + this.statKind = statKind; + this.statObject = statObject; + this.itemNames = itemNames; + + AtomicLong[] accs = new AtomicLong[itemNames.length]; + for (int i = 0; i < itemNames.length; i++) { + accs[i] = new AtomicLong(0); + } + + this.itemAccumulates = accs; + this.invokeTimes = new AtomicLong(); + this.lastTimeStamp = new AtomicLong(System.currentTimeMillis()); + } + + public void incItems(long... itemIncs) { + int len = Math.min(itemIncs.length, itemAccumulates.length); + for (int i = 0; i < len; i++) { + itemAccumulates[i].addAndGet(itemIncs[i]); + } + + invokeTimes.addAndGet(1); + lastTimeStamp.set(System.currentTimeMillis()); + + if (interceptor != null) { + interceptor.inc(itemIncs); + } + } + + public boolean allZeros() { + if (invokeTimes.get() == 0) { + return true; + } + + for (AtomicLong acc : itemAccumulates) { + if (acc.get() != 0) { + return false; + } + } + return true; + } + + public String getStatKind() { + return statKind; + } + + public String getStatObject() { + return statObject; + } + + public String[] getItemNames() { + return itemNames; + } + + public AtomicLong[] getItemAccumulates() { + return itemAccumulates; + } + + public AtomicLong getInvokeTimes() { + return invokeTimes; + } + + public AtomicLong getLastTimeStamp() { + return lastTimeStamp; + } + + public AtomicLong getItemAccumulate(String itemName) { + int index = ArrayUtils.indexOf(itemNames, itemName); + if (index < 0) { + return new AtomicLong(0); + } + return itemAccumulates[index]; + } + + /** + * get snapshot + *

+ * Warning: no guarantee of itemAccumulates consistency + * + * @return + */ + public StatisticsItem snapshot() { + StatisticsItem ret = new StatisticsItem(statKind, statObject, itemNames); + + ret.itemAccumulates = new AtomicLong[itemAccumulates.length]; + for (int i = 0; i < itemAccumulates.length; i++) { + ret.itemAccumulates[i] = new AtomicLong(itemAccumulates[i].get()); + } + + ret.invokeTimes = new AtomicLong(invokeTimes.longValue()); + ret.lastTimeStamp = new AtomicLong(lastTimeStamp.longValue()); + + return ret; + } + + /** + * subtract another StatisticsItem + * + * @param item + * @return + */ + public StatisticsItem subtract(StatisticsItem item) { + if (item == null) { + return snapshot(); + } + + if (!statKind.equals(item.statKind) || !statObject.equals(item.statObject) || !Arrays.equals(itemNames, + item.itemNames)) { + throw new IllegalArgumentException("StatisticsItem's kind, key and itemNames must be exactly the same"); + } + + StatisticsItem ret = new StatisticsItem(statKind, statObject, itemNames); + ret.invokeTimes = new AtomicLong(invokeTimes.get() - item.invokeTimes.get()); + ret.itemAccumulates = new AtomicLong[itemAccumulates.length]; + for (int i = 0; i < itemAccumulates.length; i++) { + ret.itemAccumulates[i] = new AtomicLong(itemAccumulates[i].get() - item.itemAccumulates[i].get()); + } + return ret; + } + + public Interceptor getInterceptor() { + return interceptor; + } + + public void setInterceptor(Interceptor interceptor) { + this.interceptor = interceptor; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemFormatter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemFormatter.java new file mode 100644 index 0000000000..87e301addf --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemFormatter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.util.concurrent.atomic.AtomicLong; + +public class StatisticsItemFormatter { + public String format(StatisticsItem statItem) { + final String seperator = "|"; + StringBuilder sb = new StringBuilder(); + sb.append(statItem.getStatKind()).append(seperator); + sb.append(statItem.getStatObject()).append(seperator); + for (AtomicLong acc : statItem.getItemAccumulates()) { + sb.append(acc.get()).append(seperator); + } + sb.append(statItem.getInvokeTimes()); + return sb.toString(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemPrinter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemPrinter.java new file mode 100644 index 0000000000..26c1df8ca6 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemPrinter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import org.apache.rocketmq.logging.InternalLogger; + +public class StatisticsItemPrinter { + private InternalLogger log; + + private StatisticsItemFormatter formatter; + + public StatisticsItemPrinter(StatisticsItemFormatter formatter, InternalLogger log) { + this.formatter = formatter; + this.log = log; + } + + public void log(InternalLogger log) { + this.log = log; + } + + public void formatter(StatisticsItemFormatter formatter) { + this.formatter = formatter; + } + + public void print(String prefix, StatisticsItem statItem, String... suffixs) { + StringBuilder suffix = new StringBuilder(); + for (String str : suffixs) { + suffix.append(str); + } + + if (log != null) { + log.info("{}{}{}", prefix, formatter.format(statItem), suffix.toString()); + } + // System.out.printf("%s %s%s%s\n", new Date().toString(), prefix, formatter.format(statItem), suffix.toString()); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java new file mode 100644 index 0000000000..d949b91880 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledIncrementPrinter.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class StatisticsItemScheduledIncrementPrinter extends StatisticsItemScheduledPrinter { + + private String[] tpsItemNames; + + public static final int TPS_INITIAL_DELAY = 0; + public static final int TPS_INTREVAL = 1000; + public static final String SEPERATOR = "|"; + + /** + * last snapshots of all scheduled items + */ + private final ConcurrentHashMap> lastItemSnapshots + = new ConcurrentHashMap>(); + + private final ConcurrentHashMap> sampleBriefs + = new ConcurrentHashMap>(); + + public StatisticsItemScheduledIncrementPrinter(String name, StatisticsItemPrinter printer, + ScheduledExecutorService executor, InitialDelay initialDelay, + long interval, String[] tpsItemNames, Valve valve) { + super(name, printer, executor, initialDelay, interval, valve); + this.tpsItemNames = tpsItemNames; + } + + /** + * schedule a StatisticsItem to print the Increments periodically + */ + @Override + public void schedule(final StatisticsItem item) { + setItemSampleBrief(item.getStatKind(), item.getStatObject(), new StatisticsItemSampleBrief(item, tpsItemNames)); + + // print log every ${interval} miliseconds + ScheduledFuture future = executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + if (!enabled()) { + return; + } + + StatisticsItem snapshot = item.snapshot(); + StatisticsItem lastSnapshot = getItemSnapshot(lastItemSnapshots, item.getStatKind(), + item.getStatObject()); + StatisticsItem increment = snapshot.subtract(lastSnapshot); + + Interceptor inteceptor = item.getInterceptor(); + String inteceptorStr = formatInterceptor(inteceptor); + if (inteceptor != null) { + inteceptor.reset(); + } + + StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject()); + if (brief != null && (!increment.allZeros() || printZeroLine())) { + printer.print(name, increment, inteceptorStr, brief.toString()); + } + + setItemSnapshot(lastItemSnapshots, snapshot); + + if (brief != null) { + brief.reset(); + } + } + }, getInitialDelay(), interval, TimeUnit.MILLISECONDS); + addFuture(item, future); + + // sample every TPS_INTREVAL + ScheduledFuture futureSample = executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + if (!enabled()) { + return; + } + + StatisticsItem snapshot = item.snapshot(); + StatisticsItemSampleBrief brief = getSampleBrief(item.getStatKind(), item.getStatObject()); + if (brief != null) { + brief.sample(snapshot); + } + } + }, TPS_INTREVAL, TPS_INTREVAL, TimeUnit.MILLISECONDS); + addFuture(item, futureSample); + } + + @Override + public void remove(StatisticsItem item) { + // remove task + removeAllFuture(item); + + String kind = item.getStatKind(); + String key = item.getStatObject(); + + ConcurrentHashMap lastItemMap = lastItemSnapshots.get(kind); + if (lastItemMap != null) { + lastItemMap.remove(key); + } + + ConcurrentHashMap briefMap = sampleBriefs.get(kind); + if (briefMap != null) { + briefMap.remove(key); + } + } + + private StatisticsItem getItemSnapshot( + ConcurrentHashMap> snapshots, + String kind, String key) { + ConcurrentHashMap itemMap = snapshots.get(kind); + return (itemMap != null) ? itemMap.get(key) : null; + } + + private StatisticsItemSampleBrief getSampleBrief(String kind, String key) { + ConcurrentHashMap itemMap = sampleBriefs.get(kind); + return (itemMap != null) ? itemMap.get(key) : null; + } + + private void setItemSnapshot(ConcurrentHashMap> snapshots, + StatisticsItem item) { + String kind = item.getStatKind(); + String key = item.getStatObject(); + ConcurrentHashMap itemMap = snapshots.get(kind); + if (itemMap == null) { + itemMap = new ConcurrentHashMap(); + ConcurrentHashMap oldItemMap = snapshots.putIfAbsent(kind, itemMap); + if (oldItemMap != null) { + itemMap = oldItemMap; + } + } + + itemMap.put(key, item); + } + + private void setItemSampleBrief(String kind, String key, + StatisticsItemSampleBrief brief) { + ConcurrentHashMap itemMap = sampleBriefs.get(kind); + if (itemMap == null) { + itemMap = new ConcurrentHashMap(); + ConcurrentHashMap oldItemMap = sampleBriefs.putIfAbsent(kind, itemMap); + if (oldItemMap != null) { + itemMap = oldItemMap; + } + } + + itemMap.put(key, brief); + } + + private String formatInterceptor(Interceptor interceptor) { + if (interceptor == null) { + return ""; + } + + if (interceptor instanceof StatisticsBriefInterceptor) { + StringBuilder sb = new StringBuilder(); + StatisticsBriefInterceptor briefInterceptor = (StatisticsBriefInterceptor)interceptor; + for (StatisticsBrief brief : briefInterceptor.getStatisticsBriefs()) { + long max = brief.getMax(); + long tp999 = Math.min(brief.tp999(), max); + //sb.append(SEPERATOR).append(brief.getTotal()); + sb.append(SEPERATOR).append(max); + //sb.append(SEPERATOR).append(brief.getMin()); + sb.append(SEPERATOR).append(String.format("%.2f", brief.getAvg())); + sb.append(SEPERATOR).append(tp999); + } + return sb.toString(); + } + return ""; + } + + public static class StatisticsItemSampleBrief { + private StatisticsItem lastSnapshot; + + public String[] itemNames; + public ItemSampleBrief[] briefs; + + public StatisticsItemSampleBrief(StatisticsItem statItem, String[] itemNames) { + this.lastSnapshot = statItem.snapshot(); + this.itemNames = itemNames; + this.briefs = new ItemSampleBrief[itemNames.length]; + for (int i = 0; i < itemNames.length; i++) { + this.briefs[i] = new ItemSampleBrief(); + } + } + + public synchronized void reset() { + for (ItemSampleBrief brief : briefs) { + brief.reset(); + } + } + + public synchronized void sample(StatisticsItem snapshot) { + if (snapshot == null) { + return; + } + + for (int i = 0; i < itemNames.length; i++) { + String name = itemNames[i]; + + long lastValue = lastSnapshot != null ? lastSnapshot.getItemAccumulate(name).get() : 0; + long increment = snapshot.getItemAccumulate(name).get() - lastValue; + briefs[i].sample(increment); + } + lastSnapshot = snapshot; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < briefs.length; i++) { + ItemSampleBrief brief = briefs[i]; + sb.append(SEPERATOR).append(brief.getMax()); + //sb.append(SEPERATOR).append(brief.getMin()); + sb.append(SEPERATOR).append(String.format("%.2f", brief.getAvg())); + } + return sb.toString(); + } + } + + /** + * sample brief of a item for a period of time + */ + public static class ItemSampleBrief { + private long max; + private long min; + private long total; + private long cnt; + + public ItemSampleBrief() { + reset(); + } + + public void sample(long value) { + max = Math.max(max, value); + min = Math.min(min, value); + total += value; + cnt++; + } + + public void reset() { + max = 0; + min = Long.MAX_VALUE; + total = 0; + cnt = 0; + } + + /** + * Getters + * + * @return + */ + public long getMax() { + return max; + } + + public long getMin() { + return cnt > 0 ? min : 0; + } + + public long getTotal() { + return total; + } + + public long getCnt() { + return cnt; + } + + public double getAvg() { + return cnt != 0 ? ((double)total) / cnt : 0; + } + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledPrinter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledPrinter.java new file mode 100644 index 0000000000..2f46ac5ef3 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemScheduledPrinter.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +public class StatisticsItemScheduledPrinter extends FutureHolder { + protected String name; + + protected StatisticsItemPrinter printer; + protected ScheduledExecutorService executor; + protected long interval; + protected InitialDelay initialDelay; + protected Valve valve; + + public StatisticsItemScheduledPrinter(String name, StatisticsItemPrinter printer, + ScheduledExecutorService executor, InitialDelay initialDelay, + long interval, Valve valve) { + this.name = name; + this.printer = printer; + this.executor = executor; + this.initialDelay = initialDelay; + this.interval = interval; + this.valve = valve; + } + + /** + * schedule a StatisticsItem to print all the values periodically + */ + public void schedule(final StatisticsItem statisticsItem) { + ScheduledFuture future = executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + if (enabled()) { + printer.print(name, statisticsItem); + } + } + }, getInitialDelay(), interval, TimeUnit.MILLISECONDS); + + addFuture(statisticsItem, future); + } + + public void remove(final StatisticsItem statisticsItem) { + removeAllFuture(statisticsItem); + } + + public interface InitialDelay { + /** + * Get inital delay value + * @return + */ + long get(); + } + + public interface Valve { + /** + * whether enabled + * @return + */ + boolean enabled(); + + /** + * whether print zero lines + * @return + */ + boolean printZeroLine(); + } + + protected long getInitialDelay() { + return initialDelay != null ? initialDelay.get() : 0; + } + + protected boolean enabled() { + return valve != null ? valve.enabled() : false; + } + + protected boolean printZeroLine() { + return valve != null ? valve.printZeroLine() : false; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemStateGetter.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemStateGetter.java new file mode 100644 index 0000000000..3b16d00e12 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsItemStateGetter.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +public interface StatisticsItemStateGetter { + boolean online(StatisticsItem item); +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsKindMeta.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsKindMeta.java new file mode 100644 index 0000000000..27bee19b21 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsKindMeta.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +/** + * Statistics Kind Metadata + */ +public class StatisticsKindMeta { + private String name; + private String[] itemNames; + private StatisticsItemScheduledPrinter scheduledPrinter; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String[] getItemNames() { + return itemNames; + } + + public void setItemNames(String[] itemNames) { + this.itemNames = itemNames; + } + + public StatisticsItemScheduledPrinter getScheduledPrinter() { + return scheduledPrinter; + } + + public void setScheduledPrinter(StatisticsItemScheduledPrinter scheduledPrinter) { + this.scheduledPrinter = scheduledPrinter; + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsManager.java b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsManager.java new file mode 100644 index 0000000000..b517b63159 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statistics/StatisticsManager.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statistics; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang3.tuple.Pair; +import org.apache.rocketmq.common.utils.ThreadUtils; + +public class StatisticsManager { + + /** + * Set of Statistics Kind Metadata + */ + private Map kindMetaMap; + + /** + * item names to calculate statistics brief + */ + private Pair[] briefMetas; + + /** + * Statistics + */ + private final ConcurrentHashMap> statsTable + = new ConcurrentHashMap>(); + + private static final int MAX_IDLE_TIME = 10 * 60 * 1000; + private final ScheduledExecutorService executor = ThreadUtils.newSingleThreadScheduledExecutor( + "StatisticsManagerCleaner", true); + + private StatisticsItemStateGetter statisticsItemStateGetter; + + public StatisticsManager() { + kindMetaMap = new HashMap(); + start(); + } + + public StatisticsManager(Map kindMeta) { + this.kindMetaMap = kindMeta; + start(); + } + + public void addStatisticsKindMeta(StatisticsKindMeta kindMeta) { + kindMetaMap.put(kindMeta.getName(), kindMeta); + statsTable.putIfAbsent(kindMeta.getName(), new ConcurrentHashMap(16)); + } + + public void setBriefMeta(Pair[] briefMetas) { + this.briefMetas = briefMetas; + } + + private void start() { + int maxIdleTime = MAX_IDLE_TIME; + executor.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + Iterator>> iter + = statsTable.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + String kind = entry.getKey(); + ConcurrentHashMap itemMap = entry.getValue(); + + if (itemMap == null || itemMap.isEmpty()) { + continue; + } + + HashMap tmpItemMap = new HashMap(itemMap); + for (StatisticsItem item : tmpItemMap.values()) { + // remove when expired + if (System.currentTimeMillis() - item.getLastTimeStamp().get() > MAX_IDLE_TIME + && (statisticsItemStateGetter == null || !statisticsItemStateGetter.online(item))) { + remove(item); + } + } + } + } + }, maxIdleTime, maxIdleTime / 3, TimeUnit.MILLISECONDS); + } + + /** + * Increment a StatisticsItem + * + * @param kind + * @param key + * @param itemAccumulates + */ + public boolean inc(String kind, String key, long... itemAccumulates) { + ConcurrentHashMap itemMap = statsTable.get(kind); + if (itemMap != null) { + StatisticsItem item = itemMap.get(key); + + // if not exist, create and schedule + if (item == null) { + item = new StatisticsItem(kind, key, kindMetaMap.get(kind).getItemNames()); + item.setInterceptor(new StatisticsBriefInterceptor(item, briefMetas)); + StatisticsItem oldItem = itemMap.putIfAbsent(key, item); + if (oldItem != null) { + item = oldItem; + } else { + scheduleStatisticsItem(item); + } + } + + // do increment + item.incItems(itemAccumulates); + + return true; + } + + return false; + } + + private void scheduleStatisticsItem(StatisticsItem item) { + kindMetaMap.get(item.getStatKind()).getScheduledPrinter().schedule(item); + } + + public void remove(StatisticsItem item) { + ConcurrentHashMap itemMap = statsTable.get(item.getStatKind()); + if (itemMap != null) { + itemMap.remove(item.getStatObject(), item); + } + + StatisticsKindMeta kindMeta = kindMetaMap.get(item.getStatKind()); + if (kindMeta != null && kindMeta.getScheduledPrinter() != null) { + kindMeta.getScheduledPrinter().remove(item); + } + } + + public StatisticsItemStateGetter getStatisticsItemStateGetter() { + return statisticsItemStateGetter; + } + + public void setStatisticsItemStateGetter(StatisticsItemStateGetter statisticsItemStateGetter) { + this.statisticsItemStateGetter = statisticsItemStateGetter; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/subscription/GroupForbidden.java b/common/src/main/java/org/apache/rocketmq/common/subscription/GroupForbidden.java new file mode 100644 index 0000000000..73d50c6df0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/subscription/GroupForbidden.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.subscription; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +/** + * + */ +public class GroupForbidden extends RemotingSerializable { + + private String topic; + private String group; + private Boolean readable; + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public Boolean getReadable() { + return readable; + } + + public void setReadable(Boolean readable) { + this.readable = readable; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((group == null) ? 0 : group.hashCode()); + result = prime * result + ((readable == null) ? 0 : readable.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + GroupForbidden other = (GroupForbidden) obj; + return new EqualsBuilder() + .append(topic, other.topic) + .append(group, other.group) + .append(readable, other.readable) + .isEquals(); + } + + @Override + public String toString() { + return "GroupForbidden [topic=" + topic + ", group=" + group + ", readable=" + readable + "]"; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/subscription/SubscriptionGroupConfig.java b/common/src/main/java/org/apache/rocketmq/common/subscription/SubscriptionGroupConfig.java index 8f4703f6c8..6981c236a9 100644 --- a/common/src/main/java/org/apache/rocketmq/common/subscription/SubscriptionGroupConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/subscription/SubscriptionGroupConfig.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.subscription; +import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.rocketmq.common.MixAll; public class SubscriptionGroupConfig { @@ -38,6 +39,8 @@ public class SubscriptionGroupConfig { private boolean notifyConsumerIdsChangedEnable = true; + private int groupSysFlag = 0; + public String getGroupName() { return groupName; } @@ -110,6 +113,14 @@ public class SubscriptionGroupConfig { this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable; } + public int getGroupSysFlag() { + return groupSysFlag; + } + + public void setGroupSysFlag(int groupSysFlag) { + this.groupSysFlag = groupSysFlag; + } + @Override public int hashCode() { final int prime = 31; @@ -136,28 +147,18 @@ public class SubscriptionGroupConfig { if (getClass() != obj.getClass()) return false; SubscriptionGroupConfig other = (SubscriptionGroupConfig) obj; - if (brokerId != other.brokerId) - return false; - if (consumeBroadcastEnable != other.consumeBroadcastEnable) - return false; - if (consumeEnable != other.consumeEnable) - return false; - if (consumeFromMinEnable != other.consumeFromMinEnable) - return false; - if (groupName == null) { - if (other.groupName != null) - return false; - } else if (!groupName.equals(other.groupName)) - return false; - if (retryMaxTimes != other.retryMaxTimes) - return false; - if (retryQueueNums != other.retryQueueNums) - return false; - if (whichBrokerWhenConsumeSlowly != other.whichBrokerWhenConsumeSlowly) - return false; - if (notifyConsumerIdsChangedEnable != other.notifyConsumerIdsChangedEnable) - return false; - return true; + return new EqualsBuilder() + .append(groupName, other.groupName) + .append(consumeEnable, other.consumeEnable) + .append(consumeFromMinEnable, other.consumeFromMinEnable) + .append(consumeBroadcastEnable, other.consumeBroadcastEnable) + .append(retryQueueNums, other.retryQueueNums) + .append(retryMaxTimes, other.retryMaxTimes) + .append(brokerId, other.brokerId) + .append(whichBrokerWhenConsumeSlowly, other.whichBrokerWhenConsumeSlowly) + .append(notifyConsumerIdsChangedEnable, other.notifyConsumerIdsChangedEnable) + .append(groupSysFlag, other.groupSysFlag) + .isEquals(); } @Override @@ -167,6 +168,6 @@ public class SubscriptionGroupConfig { + consumeBroadcastEnable + ", retryQueueNums=" + retryQueueNums + ", retryMaxTimes=" + retryMaxTimes + ", brokerId=" + brokerId + ", whichBrokerWhenConsumeSlowly=" + whichBrokerWhenConsumeSlowly + ", notifyConsumerIdsChangedEnable=" - + notifyConsumerIdsChangedEnable + "]"; + + notifyConsumerIdsChangedEnable + ", groupSysFlag=" + groupSysFlag + "]"; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java index d4a9d808e8..540f53ad76 100644 --- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java @@ -36,6 +36,8 @@ public class TopicValidator { public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; + public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_"; + public static final boolean[] VALID_CHAR_BIT_MAP = new boolean[128]; private static final int TOPIC_MAX_LENGTH = 127; diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/MessageUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/MessageUtils.java new file mode 100644 index 0000000000..a2affc7e90 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/MessageUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.utils; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import com.google.common.hash.Hashing; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; + +public class MessageUtils { + + public static int getShardingKeyIndex(String shardingKey, int indexSize) { + return Math.abs(Hashing.murmur3_32().hashBytes(shardingKey.getBytes()).asInt() % indexSize); + } + + public static int getShardingKeyIndexByMsg(MessageExt msg, int indexSize) { + String shardingKey = msg.getProperty(MessageConst.PROPERTY_SHARDING_KEY); + if (shardingKey == null) { + shardingKey = ""; + } + + return getShardingKeyIndex(shardingKey, indexSize); + } + + public static Set getShardingKeyIndexes(Collection msgs, int indexSize) { + Set indexSet = new HashSet(indexSize); + for (MessageExt msg : msgs) { + indexSet.add(getShardingKeyIndexByMsg(msg, indexSize)); + } + return indexSet; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/PositiveAtomicCounter.java b/common/src/main/java/org/apache/rocketmq/common/utils/PositiveAtomicCounter.java new file mode 100644 index 0000000000..105b88c577 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/PositiveAtomicCounter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.utils; + +import java.util.concurrent.atomic.AtomicInteger; + +public class PositiveAtomicCounter { + private static final int MASK = 0x7FFFFFFF; + private final AtomicInteger atom; + + + public PositiveAtomicCounter() { + atom = new AtomicInteger(0); + } + + + public final int incrementAndGet() { + final int rt = atom.incrementAndGet(); + return rt & MASK; + } + + + public int intValue() { + return atom.intValue(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java new file mode 100644 index 0000000000..72857bc4ab --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ServiceProvider.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.utils; + +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +public class ServiceProvider { + + private static final InternalLogger LOG = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + /** + * A reference to the classloader that loaded this class. It's more efficient to compute it once and cache it here. + */ + private static ClassLoader thisClassLoader; + + /** + * JDK1.3+ 'Service Provider' specification. + */ + public static final String TRANSACTION_SERVICE_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.TransactionalMessageService"; + + public static final String TRANSACTION_LISTENER_ID = "META-INF/service/org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener"; + + public static final String HA_SERVICE_ID = "META-INF/service/org.apache.rocketmq.store.ha.HAService"; + + + public static final String RPC_HOOK_ID = "META-INF/service/org.apache.rocketmq.remoting.RPCHook"; + + + public static final String ACL_VALIDATOR_ID = "META-INF/service/org.apache.rocketmq.acl.AccessValidator"; + + + + static { + thisClassLoader = getClassLoader(ServiceProvider.class); + } + + /** + * Returns a string that uniquely identifies the specified object, including its class. + *

+ * The returned string is of form "classname@hashcode", ie is the same as the return value of the Object.toString() method, but works even when the specified object's class has overidden the toString method. + * + * @param o may be null. + * @return a string of form classname@hashcode, or "null" if param o is null. + */ + protected static String objectId(Object o) { + if (o == null) { + return "null"; + } else { + return o.getClass().getName() + "@" + System.identityHashCode(o); + } + } + + protected static ClassLoader getClassLoader(Class clazz) { + try { + return clazz.getClassLoader(); + } catch (SecurityException e) { + LOG.error("Unable to get classloader for class {} due to security restrictions !", + clazz, e.getMessage()); + throw e; + } + } + + protected static ClassLoader getContextClassLoader() { + ClassLoader classLoader = null; + try { + classLoader = Thread.currentThread().getContextClassLoader(); + } catch (SecurityException ex) { + /** + * The getContextClassLoader() method throws SecurityException when the context + * class loader isn't an ancestor of the calling class's class + * loader, or if security permissions are restricted. + */ + } + return classLoader; + } + + protected static InputStream getResourceAsStream(ClassLoader loader, String name) { + if (loader != null) { + return loader.getResourceAsStream(name); + } else { + return ClassLoader.getSystemResourceAsStream(name); + } + } + + public static List load(String name, Class clazz) { + LOG.info("Looking for a resource file of name [{}] ...", name); + List services = new ArrayList(); + try { + ArrayList names = new ArrayList(); + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + if (is != null) { + BufferedReader reader; + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + while (serviceName != null && !"".equals(serviceName)) { + LOG.info( + "Creating an instance as specified by file {} which was present in the path of the context classloader.", + name); + if (!names.contains(serviceName)) { + names.add(serviceName); + } + + services.add((T)initService(getContextClassLoader(), serviceName, clazz)); + + serviceName = reader.readLine(); + } + reader.close(); + } else { + // is == null + LOG.warn("No resource file with name [{}] found.", name); + } + } catch (Exception e) { + LOG.error("Error occured when looking for resource file " + name, e); + } + return services; + } + + public static T loadClass(String name, Class clazz) { + final InputStream is = getResourceAsStream(getContextClassLoader(), name); + if (is != null) { + BufferedReader reader; + try { + try { + reader = new BufferedReader(new InputStreamReader(is, "UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + reader = new BufferedReader(new InputStreamReader(is)); + } + String serviceName = reader.readLine(); + reader.close(); + if (serviceName != null && !"".equals(serviceName)) { + return initService(getContextClassLoader(), serviceName, clazz); + } else { + LOG.warn("ServiceName is empty!"); + return null; + } + } catch (Exception e) { + LOG.warn("Error occurred when looking for resource file " + name, e); + } + } + return null; + } + + protected static T initService(ClassLoader classLoader, String serviceName, Class clazz) { + Class serviceClazz = null; + try { + if (classLoader != null) { + try { + // Warning: must typecast here & allow exception to be generated/caught & recast properly + serviceClazz = classLoader.loadClass(serviceName); + if (clazz.isAssignableFrom(serviceClazz)) { + LOG.info("Loaded class {} from classloader {}", serviceClazz.getName(), + objectId(classLoader)); + } else { + // This indicates a problem with the ClassLoader tree. An incompatible ClassLoader was used to load the implementation. + LOG.error( + "Class {} loaded from classloader {} does not extend {} as loaded by this classloader.", + new Object[] {serviceClazz.getName(), + objectId(serviceClazz.getClassLoader()), clazz.getName()}); + } + return (T)serviceClazz.newInstance(); + } catch (ClassNotFoundException ex) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn("Unable to locate any class {} via classloader", serviceName, + objectId(classLoader)); + throw ex; + } + // Ignore exception, continue + } catch (NoClassDefFoundError e) { + if (classLoader == thisClassLoader) { + // Nothing more to try, onwards. + LOG.warn( + "Class {} cannot be loaded via classloader {}.it depends on some other class that cannot be found.", + serviceClazz, objectId(classLoader)); + throw e; + } + // Ignore exception, continue + } + } + } catch (Exception e) { + LOG.error("Unable to init service.", e); + } + return (T)serviceClazz; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java index 13c029396b..61ca4a8f2e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java @@ -30,7 +30,7 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; public final class ThreadUtils { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { @@ -102,7 +102,7 @@ public final class ThreadUtils { thread.setDaemon(daemon); thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { public void uncaughtException(Thread t, Throwable e) { - log.error("Uncaught exception in thread '" + t.getName() + "':", e); + LOGGER.error("Uncaught exception in thread '" + t.getName() + "':", e); } }); return thread; @@ -153,7 +153,7 @@ public final class ThreadUtils { executor.shutdownNow(); // Wait a while for tasks to respond to being cancelled. if (!executor.awaitTermination(timeout, timeUnit)) { - log.warn(String.format("%s didn't terminate!", executor)); + LOGGER.warn(String.format("%s didn't terminate!", executor)); } } } catch (InterruptedException ie) { @@ -164,6 +164,17 @@ public final class ThreadUtils { } } + /** + * Shutdown the specific ExecutorService + * + * @param executorService the executor + */ + public static void shutdown(ExecutorService executorService) { + if (executorService != null) { + executorService.shutdown(); + } + } + /** * A constructor to stop this class being constructed. */ diff --git a/common/src/test/java/org/apache/rocketmq/common/DataVersionTest.java b/common/src/test/java/org/apache/rocketmq/common/DataVersionTest.java index f4d14e5614..d6cc4b005d 100644 --- a/common/src/test/java/org/apache/rocketmq/common/DataVersionTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/DataVersionTest.java @@ -67,4 +67,11 @@ public class DataVersionTest { other.setTimestamp(dataVersion.getTimestamp()); Assert.assertTrue(dataVersion.equals(other)); } + + @Test + public void testEncode() { + DataVersion dataVersion = new DataVersion(); + Assert.assertTrue(dataVersion.encode().length > 0); + Assert.assertNotNull(dataVersion.toJson()); + } } \ No newline at end of file diff --git a/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java index 428a9289cd..73703c02d3 100644 --- a/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; -import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; + import static org.junit.Assert.assertEquals; import org.junit.Test; diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java index 7ef7eaca2a..d1ed4bb451 100644 --- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java @@ -19,13 +19,11 @@ package org.apache.rocketmq.common; import java.net.InetAddress; import java.net.UnknownHostException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; - -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableList; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -174,4 +172,21 @@ public class UtilAllTest { '}'; } } + + @Test + public void testCleanBuffer() { + UtilAll.cleanBuffer(null); + UtilAll.cleanBuffer(ByteBuffer.allocate(10)); + UtilAll.cleanBuffer(ByteBuffer.allocate(0)); + } + + @Test(expected = NoSuchMethodException.class) + public void testMethod() throws NoSuchMethodException { + UtilAll.method(new Object(), "noMethod", null); + } + + @Test(expected = IllegalStateException.class) + public void testInvoke() throws Exception { + UtilAll.invoke(new Object(), "noMethod"); + } } diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java index 5af7345297..c57c75889d 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageDecoderTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.message; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.junit.Test; @@ -163,6 +164,8 @@ public class MessageDecoderTest { messageExt.putUserProperty("b", "hello"); messageExt.putUserProperty("c", "3.14"); + messageExt.setBodyCRC(UtilAll.crc32(messageExt.getBody())); + byte[] msgBytes = new byte[0]; try { msgBytes = MessageDecoder.encode(messageExt, false); @@ -174,7 +177,7 @@ public class MessageDecoderTest { ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length); byteBuffer.put(msgBytes); - byteBuffer.clear(); + byteBuffer.flip(); MessageExt decodedMsg = MessageDecoder.decode(byteBuffer); assertThat(decodedMsg).isNotNull(); @@ -222,6 +225,8 @@ public class MessageDecoderTest { messageExt.putUserProperty("b", "hello"); messageExt.putUserProperty("c", "3.14"); + messageExt.setBodyCRC(UtilAll.crc32(messageExt.getBody())); + byte[] msgBytes = new byte[0]; try { msgBytes = MessageDecoder.encode(messageExt, false); @@ -233,7 +238,7 @@ public class MessageDecoderTest { ByteBuffer byteBuffer = ByteBuffer.allocate(msgBytes.length); byteBuffer.put(msgBytes); - byteBuffer.clear(); + byteBuffer.flip(); MessageExt decodedMsg = MessageDecoder.decode(byteBuffer); assertThat(decodedMsg).isNotNull(); diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageTest.java index c867360f85..c950970980 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageTest.java @@ -20,7 +20,6 @@ import org.junit.Assert; import org.junit.Test; import static org.apache.rocketmq.common.message.MessageConst.PROPERTY_TRACE_SWITCH; -import static org.junit.Assert.*; public class MessageTest { @Test(expected = RuntimeException.class) diff --git a/common/src/test/java/org/apache/rocketmq/common/protocol/body/KVTableTest.java b/common/src/test/java/org/apache/rocketmq/common/protocol/body/KVTableTest.java index 836733c506..6a39cee987 100644 --- a/common/src/test/java/org/apache/rocketmq/common/protocol/body/KVTableTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/protocol/body/KVTableTest.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.common.protocol.body; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.junit.Assert; import org.junit.Test; import java.util.HashMap; diff --git a/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java index 4e93275ea9..6296dd586b 100644 --- a/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java @@ -21,11 +21,9 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.google.common.collect.ImmutableList; -import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.junit.Assert; import org.junit.Test; -import org.junit.experimental.theories.suppliers.TestedOn; import java.util.Map; diff --git a/namesrv/pom.xml b/namesrv/pom.xml index 11ddfb76b5..0b04d50145 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -44,13 +44,24 @@ ch.qos.logback logback-classic + + ch.qos.logback + logback-core + org.slf4j slf4j-api - com.google.guava - guava + org.openjdk.jmh + jmh-core + 1.19 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.19 test diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java index a6654f271c..d02e6c0ad3 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java @@ -16,58 +16,92 @@ */ package org.apache.rocketmq.namesrv; +import java.util.Arrays; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RunnableFuture; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.future.FutureTaskExt; +import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; +import org.apache.rocketmq.namesrv.processor.ClientRequestProcessor; import org.apache.rocketmq.namesrv.processor.ClusterTestRequestProcessor; import org.apache.rocketmq.namesrv.processor.DefaultRequestProcessor; import org.apache.rocketmq.namesrv.routeinfo.BrokerHousekeepingService; import org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager; +import org.apache.rocketmq.remoting.RemotingClient; import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.common.TlsMode; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.netty.TlsSystemConfig; import org.apache.rocketmq.srvutil.FileWatchService; public class NamesrvController { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + private static final InternalLogger WATER_MARK_LOG = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_WATER_MARK_LOGGER_NAME); private final NamesrvConfig namesrvConfig; private final NettyServerConfig nettyServerConfig; + private final NettyClientConfig nettyClientConfig; - private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( - "NSScheduledThread")); + private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, + new BasicThreadFactory.Builder() + .namingPattern("NSScheduledThread") + .daemon(true) + .build()); + private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1, + new BasicThreadFactory.Builder() + .namingPattern("NSScanScheduledThread") + .daemon(true) + .build()); private final KVConfigManager kvConfigManager; private final RouteInfoManager routeInfoManager; + private RemotingClient remotingClient; private RemotingServer remotingServer; private BrokerHousekeepingService brokerHousekeepingService; - private ExecutorService remotingExecutor; + private ExecutorService defaultExecutor; + private ExecutorService clientRequestExecutor; + + private BlockingQueue defaultThreadPoolQueue; + private BlockingQueue clientRequestThreadPoolQueue; private Configuration configuration; private FileWatchService fileWatchService; public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig) { + this(namesrvConfig, nettyServerConfig, new NettyClientConfig()); + } + + public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig, NettyClientConfig nettyClientConfig) { this.namesrvConfig = namesrvConfig; this.nettyServerConfig = nettyServerConfig; + this.nettyClientConfig = nettyClientConfig; this.kvConfigManager = new KVConfigManager(this); - this.routeInfoManager = new RouteInfoManager(); this.brokerHousekeepingService = new BrokerHousekeepingService(this); + this.routeInfoManager = new RouteInfoManager(namesrvConfig, this); this.configuration = new Configuration( - log, + LOGGER, this.namesrvConfig, this.nettyServerConfig ); this.configuration.setStorePathFromConfig(this.namesrvConfig, "configStorePath"); @@ -79,26 +113,51 @@ public class NamesrvController { this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.brokerHousekeepingService); - this.remotingExecutor = - Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), new ThreadFactoryImpl("RemotingExecutorThread_")); + this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity()); + this.defaultExecutor = new ThreadPoolExecutor( + this.namesrvConfig.getDefaultThreadPoolNums(), + this.namesrvConfig.getDefaultThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.defaultThreadPoolQueue, + new ThreadFactoryImpl("RemotingExecutorThread_")) { + @Override + protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { + return new FutureTaskExt(runnable, value); + } + }; + + this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity()); + this.clientRequestExecutor = new ThreadPoolExecutor( + this.namesrvConfig.getClientRequestThreadPoolNums(), + this.namesrvConfig.getClientRequestThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.clientRequestThreadPoolQueue, + new ThreadFactoryImpl("ClientRequestExecutorThread_")) { + @Override + protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { + return new FutureTaskExt(runnable, value); + } + }; + this.remotingClient = new NettyRemotingClient(this.nettyClientConfig); + this.remotingClient.updateNameServerAddressList(Arrays.asList(RemotingUtil.getLocalAddress() + ":" + this.nettyServerConfig.getListenPort())); this.registerProcessor(); - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + this.scanExecutorService.scheduleAtFixedRate(NamesrvController.this.routeInfoManager::scanNotActiveBroker, + 5, this.namesrvConfig.getScanNotActiveBrokerInterval(), TimeUnit.MILLISECONDS); - @Override - public void run() { - NamesrvController.this.routeInfoManager.scanNotActiveBroker(); + this.scheduledExecutorService.scheduleAtFixedRate(NamesrvController.this.kvConfigManager::printAllPeriodically, + 1, 10, TimeUnit.MINUTES); + + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + NamesrvController.this.printWaterMark(); + } catch (Throwable e) { + LOGGER.error("printWaterMark error.", e); } - }, 5, 10, TimeUnit.SECONDS); - - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { - - @Override - public void run() { - NamesrvController.this.kvConfigManager.printAllPeriodically(); - } - }, 1, 10, TimeUnit.MINUTES); + }, 10, 1, TimeUnit.SECONDS); if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { // Register a listener to reload SslContext @@ -111,10 +170,11 @@ public class NamesrvController { }, new FileWatchService.Listener() { boolean certChanged, keyChanged = false; + @Override public void onChanged(String path) { if (path.equals(TlsSystemConfig.tlsServerTrustCertPath)) { - log.info("The trust certificate changed, reload the ssl context"); + LOGGER.info("The trust certificate changed, reload the ssl context"); reloadServerSslContext(); } if (path.equals(TlsSystemConfig.tlsServerCertPath)) { @@ -124,46 +184,82 @@ public class NamesrvController { keyChanged = true; } if (certChanged && keyChanged) { - log.info("The certificate and private key changed, reload the ssl context"); + LOGGER.info("The certificate and private key changed, reload the ssl context"); certChanged = keyChanged = false; reloadServerSslContext(); } } + private void reloadServerSslContext() { ((NettyRemotingServer) remotingServer).loadSslContext(); } }); } catch (Exception e) { - log.warn("FileWatchService created error, can't load the certificate dynamically"); + LOGGER.warn("FileWatchService created error, can't load the certificate dynamically"); } } return true; } + private void printWaterMark() { + WATER_MARK_LOG.info("[WATERMARK] ClientQueueSize:{} ClientQueueSlowTime:{} " + + "DefaultQueueSize:{} DefaultQueueSlowTime:{}", + this.clientRequestThreadPoolQueue.size(), headSlowTimeMills(this.clientRequestThreadPoolQueue), + this.defaultThreadPoolQueue.size(), headSlowTimeMills(this.defaultThreadPoolQueue)); + } + + private long headSlowTimeMills(BlockingQueue q) { + long slowTimeMills = 0; + final Runnable firstRunnable = q.peek(); + + if (firstRunnable instanceof FutureTaskExt) { + final Runnable inner = ((FutureTaskExt) firstRunnable).getRunnable(); + if (inner instanceof RequestTask) { + slowTimeMills = System.currentTimeMillis() - ((RequestTask) inner).getCreateTimestamp(); + } + } + + if (slowTimeMills < 0) { + slowTimeMills = 0; + } + + return slowTimeMills; + } + private void registerProcessor() { if (namesrvConfig.isClusterTest()) { this.remotingServer.registerDefaultProcessor(new ClusterTestRequestProcessor(this, namesrvConfig.getProductEnvName()), - this.remotingExecutor); + this.defaultExecutor); } else { + // Support get route info only temporarily + ClientRequestProcessor clientRequestProcessor = new ClientRequestProcessor(this); + this.remotingServer.registerProcessor(RequestCode.GET_ROUTEINFO_BY_TOPIC, clientRequestProcessor, this.clientRequestExecutor); - this.remotingServer.registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); + this.remotingServer.registerDefaultProcessor(new DefaultRequestProcessor(this), this.defaultExecutor); } } public void start() throws Exception { this.remotingServer.start(); + this.remotingClient.start(); if (this.fileWatchService != null) { this.fileWatchService.start(); } + + this.routeInfoManager.start(); } public void shutdown() { + this.remotingClient.shutdown(); this.remotingServer.shutdown(); - this.remotingExecutor.shutdown(); + this.defaultExecutor.shutdown(); + this.clientRequestExecutor.shutdown(); this.scheduledExecutorService.shutdown(); + this.scanExecutorService.shutdown(); + this.routeInfoManager.shutdown(); if (this.fileWatchService != null) { this.fileWatchService.shutdown(); @@ -190,6 +286,10 @@ public class NamesrvController { return remotingServer; } + public RemotingClient getRemotingClient() { + return remotingClient; + } + public void setRemotingServer(RemotingServer remotingServer) { this.remotingServer = remotingServer; } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java index d0061289dd..715f23872b 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvStartup.java @@ -35,6 +35,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.srvutil.ServerUtil; @@ -81,6 +82,7 @@ public class NamesrvStartup { final NamesrvConfig namesrvConfig = new NamesrvConfig(); final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + final NettyClientConfig nettyClientConfig = new NettyClientConfig(); nettyServerConfig.setListenPort(9876); if (commandLine.hasOption('c')) { String file = commandLine.getOptionValue('c'); @@ -90,6 +92,7 @@ public class NamesrvStartup { properties.load(in); MixAll.properties2Object(properties, namesrvConfig); MixAll.properties2Object(properties, nettyServerConfig); + MixAll.properties2Object(properties, nettyClientConfig); namesrvConfig.setConfigStorePath(file); @@ -99,9 +102,9 @@ public class NamesrvStartup { } if (commandLine.hasOption('p')) { - InternalLogger console = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_CONSOLE_NAME); - MixAll.printObjectProperties(console, namesrvConfig); - MixAll.printObjectProperties(console, nettyServerConfig); + MixAll.printObjectProperties(null, namesrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + MixAll.printObjectProperties(null, nettyClientConfig); System.exit(0); } @@ -123,7 +126,7 @@ public class NamesrvStartup { MixAll.printObjectProperties(log, namesrvConfig); MixAll.printObjectProperties(log, nettyServerConfig); - final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig); + final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig, nettyClientConfig); // remember all configs to prevent discard controller.getConfiguration().registerConfig(properties); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java new file mode 100644 index 0000000000..6f624e31bc --- /dev/null +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClientRequestProcessor.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.processor; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.namesrv.NamesrvUtil; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.protocol.route.TopicRouteDatas; +import org.apache.rocketmq.namesrv.NamesrvController; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class ClientRequestProcessor implements NettyRequestProcessor { + protected NamesrvController namesrvController; + + public ClientRequestProcessor(final NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, + final RemotingCommand request) throws Exception { + return this.getRouteInfoByTopic(ctx, request); + } + + public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetRouteInfoRequestHeader requestHeader = + (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); + + if (requestHeader.getTopic().indexOf(GetRouteInfoRequestHeader.split) < 0) { + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + + if (topicRouteData != null) { + if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { + String orderTopicConf = + this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, + requestHeader.getTopic()); + + // Get a shallow clone of route data to modify order topic conf + topicRouteData = topicRouteData.cloneTopicRouteData(); + topicRouteData.setOrderTopicConf(orderTopicConf); + } + + byte[] content = topicRouteData.encode(); + response.setBody(content); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("No topic route info in name server for the topic: " + requestHeader.getTopic() + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + + String[] topics = requestHeader.getTopic().split(String.valueOf(GetRouteInfoRequestHeader.split)); + TopicRouteDatas topicRouteDatas = new TopicRouteDatas(); + + for (String topic : topics) { + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(topic); + if (topicRouteData == null) { + continue; + } + + if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { + String orderTopicConf = + this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, topic); + // Get a shallow clone of route data to modify order topic conf + topicRouteData = topicRouteData.cloneTopicRouteData(); + topicRouteData.setOrderTopicConf(orderTopicConf); + } + + topicRouteDatas.getTopics().put(topic, topicRouteData); + } + response.setBody(topicRouteDatas.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + @Override public boolean rejectRequest() { + return false; + } +} diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java index a58a3b97fc..e5ca2d30b1 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java @@ -31,7 +31,7 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -public class ClusterTestRequestProcessor extends DefaultRequestProcessor { +public class ClusterTestRequestProcessor extends ClientRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); private final DefaultMQAdminExt adminExt; private final String productEnvName; diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index bde03489d3..a293683676 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.namesrv.processor; import io.netty.channel.ChannelHandlerContext; import java.io.UnsupportedEncodingException; import java.util.Properties; +import java.util.List; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MQVersion; @@ -26,9 +27,14 @@ import org.apache.rocketmq.common.MQVersion.Version; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.protocol.body.BrokerMemberGroup; +import org.apache.rocketmq.common.protocol.body.GetBrokerMemberGroupResponseBody; +import org.apache.rocketmq.common.protocol.body.GetRemoteClientConfigBody; +import org.apache.rocketmq.common.protocol.header.GetBrokerMemberGroupRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.BrokerHeartbeatRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.RegisterTopicRequestHeader; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.NamesrvUtil; @@ -43,7 +49,6 @@ import org.apache.rocketmq.common.protocol.header.namesrv.DeleteTopicFromNamesrv import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVListByNamespaceRequestHeader; -import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionResponseHeader; @@ -56,11 +61,11 @@ import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; -public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { +public class DefaultRequestProcessor implements NettyRequestProcessor { private static InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); protected final NamesrvController namesrvController; @@ -80,7 +85,6 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen request); } - switch (request.getCode()) { case RequestCode.PUT_KV_CONFIG: return this.putKVConfig(ctx, request); @@ -89,18 +93,15 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen case RequestCode.DELETE_KV_CONFIG: return this.deleteKVConfig(ctx, request); case RequestCode.QUERY_DATA_VERSION: - return queryBrokerTopicConfig(ctx, request); + return this.queryBrokerTopicConfig(ctx, request); case RequestCode.REGISTER_BROKER: - Version brokerVersion = MQVersion.value2Version(request.getVersion()); - if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) { - return this.registerBrokerWithFilterServer(ctx, request); - } else { - return this.registerBroker(ctx, request); - } + return this.registerBroker(ctx, request); case RequestCode.UNREGISTER_BROKER: return this.unregisterBroker(ctx, request); - case RequestCode.GET_ROUTEINFO_BY_TOPIC: - return this.getRouteInfoByTopic(ctx, request); + case RequestCode.BROKER_HEARTBEAT: + return this.brokerHeartbeat(ctx, request); + case RequestCode.GET_BROKER_MEMBER_GROUP: + return this.getBrokerMemberGroup(ctx, request); case RequestCode.GET_BROKER_CLUSTER_INFO: return this.getBrokerClusterInfo(ctx, request); case RequestCode.WIPE_WRITE_PERM_OF_BROKER: @@ -108,9 +109,11 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen case RequestCode.ADD_WRITE_PERM_OF_BROKER: return this.addWritePermOfBroker(ctx, request); case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER: - return getAllTopicListFromNameserver(ctx, request); + return this.getAllTopicListFromNameserver(ctx, request); case RequestCode.DELETE_TOPIC_IN_NAMESRV: - return deleteTopicInNamesrv(ctx, request); + return this.deleteTopicInNamesrv(ctx, request); + case RequestCode.REGISTER_TOPIC_IN_NAMESRV: + return this.registerTopicToNamesrv(ctx, request); case RequestCode.GET_KVLIST_BY_NAMESPACE: return this.getKVListByNamespace(ctx, request); case RequestCode.GET_TOPICS_BY_CLUSTER: @@ -127,14 +130,15 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return this.updateConfig(ctx, request); case RequestCode.GET_NAMESRV_CONFIG: return this.getConfig(ctx, request); + case RequestCode.GET_CLIENT_CONFIG: + return this.getClientConfigs(ctx, request); default: - break; + String error = " request type " + request.getCode() + " not supported"; + return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); } - return null; } - @Override - public boolean rejectRequest() { + @Override public boolean rejectRequest() { return false; } @@ -200,8 +204,8 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return response; } - public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, RemotingCommand request) - throws RemotingCommandException { + public RemotingCommand registerBroker(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader(); final RegisterBrokerRequestHeader requestHeader = @@ -213,6 +217,60 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return response; } + TopicConfigSerializeWrapper topicConfigWrapper = null; + List filterServerList = null; + + Version brokerVersion = MQVersion.value2Version(request.getVersion()); + if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) { + final RegisterBrokerBody registerBrokerBody = extractRegisterBrokerBodyFromRequest(request, requestHeader); + topicConfigWrapper = registerBrokerBody.getTopicConfigSerializeWrapper(); + filterServerList = registerBrokerBody.getFilterServerList(); + } else { + // RegisterBrokerBody of old version only contains TopicConfig. + topicConfigWrapper = extractRegisterTopicConfigFromRequest(request); + } + + RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker( + requestHeader.getClusterName(), + requestHeader.getBrokerAddr(), + requestHeader.getBrokerName(), + requestHeader.getBrokerId(), + requestHeader.getHaServerAddr(), + requestHeader.getHeartbeatTimeoutMillis(), + requestHeader.getEnableActingMaster(), + topicConfigWrapper, + filterServerList, + ctx.channel() + ); + + responseHeader.setHaServerAddr(result.getHaServerAddr()); + responseHeader.setMasterAddr(result.getMasterAddr()); + + if (this.namesrvController.getNamesrvConfig().isReturnOrderTopicConfigToBroker()) { + byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); + response.setBody(jsonValue); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private TopicConfigSerializeWrapper extractRegisterTopicConfigFromRequest(final RemotingCommand request) { + TopicConfigSerializeWrapper topicConfigWrapper; + if (request.getBody() != null) { + topicConfigWrapper = TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class); + } else { + topicConfigWrapper = new TopicConfigSerializeWrapper(); + topicConfigWrapper.getDataVersion().setCounter(new AtomicLong(0)); + topicConfigWrapper.getDataVersion().setTimestamp(0L); + topicConfigWrapper.getDataVersion().setStateVersion(0L); + } + return topicConfigWrapper; + } + + private RegisterBrokerBody extractRegisterBrokerBodyFromRequest(RemotingCommand request, + RegisterBrokerRequestHeader requestHeader) throws RemotingCommandException { RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); if (request.getBody() != null) { @@ -223,27 +281,25 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen } } else { registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setCounter(new AtomicLong(0)); - registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestamp(0); + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestamp(0L); + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setStateVersion(0L); } + return registerBrokerBody; + } - RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker( - requestHeader.getClusterName(), - requestHeader.getBrokerAddr(), - requestHeader.getBrokerName(), - requestHeader.getBrokerId(), - requestHeader.getHaServerAddr(), - registerBrokerBody.getTopicConfigSerializeWrapper(), - registerBrokerBody.getFilterServerList(), - ctx.channel()); + private RemotingCommand getBrokerMemberGroup(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + GetBrokerMemberGroupRequestHeader requestHeader = (GetBrokerMemberGroupRequestHeader) request.decodeCommandCustomHeader(GetBrokerMemberGroupRequestHeader.class); - responseHeader.setHaServerAddr(result.getHaServerAddr()); - responseHeader.setMasterAddr(result.getMasterAddr()); + BrokerMemberGroup memberGroup = this.namesrvController.getRouteInfoManager() + .getBrokerMemberGroup(requestHeader.getClusterName(), requestHeader.getBrokerName()); - byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); - response.setBody(jsonValue); + GetBrokerMemberGroupResponseBody responseBody = new GetBrokerMemberGroupResponseBody(); + responseBody.setBrokerMemberGroup(memberGroup); + RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); + response.setBody(responseBody.encode()); return response; } @@ -267,13 +323,13 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final QueryDataVersionRequestHeader requestHeader = (QueryDataVersionRequestHeader) request.decodeCommandCustomHeader(QueryDataVersionRequestHeader.class); DataVersion dataVersion = DataVersion.decode(request.getBody(), DataVersion.class); + String clusterName = requestHeader.getClusterName(); + String brokerAddr = requestHeader.getBrokerAddr(); - Boolean changed = this.namesrvController.getRouteInfoManager().isBrokerTopicConfigChanged(requestHeader.getBrokerAddr(), dataVersion); - if (!changed) { - this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(requestHeader.getBrokerAddr()); - } + Boolean changed = this.namesrvController.getRouteInfoManager().isBrokerTopicConfigChanged(clusterName, brokerAddr, dataVersion); + this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(clusterName, brokerAddr); - DataVersion nameSeverDataVersion = this.namesrvController.getRouteInfoManager().queryBrokerTopicConfig(requestHeader.getBrokerAddr()); + DataVersion nameSeverDataVersion = this.namesrvController.getRouteInfoManager().queryBrokerTopicConfig(clusterName, brokerAddr); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); @@ -284,92 +340,31 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return response; } - public RemotingCommand registerBroker(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - final RemotingCommand response = RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); - final RegisterBrokerResponseHeader responseHeader = (RegisterBrokerResponseHeader) response.readCustomHeader(); - final RegisterBrokerRequestHeader requestHeader = - (RegisterBrokerRequestHeader) request.decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); - - if (!checksum(ctx, request, requestHeader)) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("crc32 not match"); - return response; - } - - TopicConfigSerializeWrapper topicConfigWrapper; - if (request.getBody() != null) { - topicConfigWrapper = TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class); - } else { - topicConfigWrapper = new TopicConfigSerializeWrapper(); - topicConfigWrapper.getDataVersion().setCounter(new AtomicLong(0)); - topicConfigWrapper.getDataVersion().setTimestamp(0); - } - - RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker( - requestHeader.getClusterName(), - requestHeader.getBrokerAddr(), - requestHeader.getBrokerName(), - requestHeader.getBrokerId(), - requestHeader.getHaServerAddr(), - topicConfigWrapper, - null, - ctx.channel() - ); - - responseHeader.setHaServerAddr(result.getHaServerAddr()); - responseHeader.setMasterAddr(result.getMasterAddr()); - - byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); - response.setBody(jsonValue); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - return response; - } - public RemotingCommand unregisterBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final UnRegisterBrokerRequestHeader requestHeader = (UnRegisterBrokerRequestHeader) request.decodeCommandCustomHeader(UnRegisterBrokerRequestHeader.class); - this.namesrvController.getRouteInfoManager().unregisterBroker( - requestHeader.getClusterName(), - requestHeader.getBrokerAddr(), - requestHeader.getBrokerName(), - requestHeader.getBrokerId()); + if (!this.namesrvController.getRouteInfoManager().submitUnRegisterBrokerRequest(requestHeader)) { + log.warn("Couldn't submit the unregister broker request to handler, broker info: {}", requestHeader); + } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; } - public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, + public RemotingCommand brokerHeartbeat(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetRouteInfoRequestHeader requestHeader = - (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); + final BrokerHeartbeatRequestHeader requestHeader = + (BrokerHeartbeatRequestHeader) request.decodeCommandCustomHeader(BrokerHeartbeatRequestHeader.class); - TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + this.namesrvController.getRouteInfoManager().updateBrokerInfoUpdateTimestamp(requestHeader.getClusterName(), requestHeader.getBrokerAddr()); - if (topicRouteData != null) { - if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { - String orderTopicConf = - this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, - requestHeader.getTopic()); - topicRouteData.setOrderTopicConf(orderTopicConf); - } - - byte[] content = topicRouteData.encode(); - response.setBody(content); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - return response; - } - - response.setCode(ResponseCode.TOPIC_NOT_EXIST); - response.setRemark("No topic route info in name server for the topic: " + requestHeader.getTopic() - + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); return response; } @@ -395,9 +390,9 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen if (ctx != null) { log.info("wipe write perm of broker[{}], client: {}, {}", - requestHeader.getBrokerName(), - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - wipeTopicCnt); + requestHeader.getBrokerName(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + wipeTopicCnt); } responseHeader.setWipeTopicCount(wipeTopicCnt); @@ -406,7 +401,8 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return response; } - private RemotingCommand addWritePermOfBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + private RemotingCommand addWritePermOfBroker(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(AddWritePermOfBrokerResponseHeader.class); final AddWritePermOfBrokerResponseHeader responseHeader = (AddWritePermOfBrokerResponseHeader) response.readCustomHeader(); final AddWritePermOfBrokerRequestHeader requestHeader = (AddWritePermOfBrokerRequestHeader) request.decodeCommandCustomHeader(AddWritePermOfBrokerRequestHeader.class); @@ -414,9 +410,9 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen int addTopicCnt = this.namesrvController.getRouteInfoManager().addWritePermOfBrokerByLock(requestHeader.getBrokerName()); log.info("add write perm of broker[{}], client: {}, {}", - requestHeader.getBrokerName(), - RemotingHelper.parseChannelRemoteAddr(ctx.channel()), - addTopicCnt); + requestHeader.getBrokerName(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), + addTopicCnt); responseHeader.setAddTopicCount(addTopicCnt); response.setCode(ResponseCode.SUCCESS); @@ -426,10 +422,33 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen private RemotingCommand getAllTopicListFromNameserver(ChannelHandlerContext ctx, RemotingCommand request) { final RemotingCommand response = RemotingCommand.createResponseCommand(null); + boolean enableAllTopicList = namesrvController.getNamesrvConfig().isEnableAllTopicList(); + log.warn("getAllTopicListFromNameserver {} enable {}", ctx.channel().remoteAddress(), enableAllTopicList); + if (enableAllTopicList) { + byte[] body = this.namesrvController.getRouteInfoManager().getAllTopicList(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("disable"); + } - byte[] body = this.namesrvController.getRouteInfoManager().getAllTopicList(); + return response; + } + + private RemotingCommand registerTopicToNamesrv(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + final RegisterTopicRequestHeader requestHeader = + (RegisterTopicRequestHeader) request.decodeCommandCustomHeader(RegisterTopicRequestHeader.class); + + TopicRouteData topicRouteData = TopicRouteData.decode(request.getBody(), TopicRouteData.class); + if (topicRouteData != null && topicRouteData.getQueueDatas() != null && !topicRouteData.getQueueDatas().isEmpty()) { + this.namesrvController.getRouteInfoManager().registerTopic(requestHeader.getTopic(), topicRouteData.getQueueDatas()); + } - response.setBody(body); response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -441,7 +460,12 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final DeleteTopicFromNamesrvRequestHeader requestHeader = (DeleteTopicFromNamesrvRequestHeader) request.decodeCommandCustomHeader(DeleteTopicFromNamesrvRequestHeader.class); - this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic()); + if (requestHeader.getClusterName() != null + && !requestHeader.getClusterName().isEmpty()) { + this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic(), requestHeader.getClusterName()); + } else { + this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic()); + } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); @@ -585,4 +609,25 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen return response; } + private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class); + + String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys()); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } catch (UnsupportedEncodingException e) { + log.error("getConfig error, ", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BatchUnRegisterService.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BatchUnRegisterService.java new file mode 100644 index 0000000000..81460e7b7b --- /dev/null +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BatchUnRegisterService.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.routeinfo; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +/** + * BatchUnRegisterProcessor provides a mechanism to unregister broker in batch manner, which can speed up broker offline + * process. + */ +public class BatchUnRegisterService extends ServiceThread { + private final RouteInfoManager routeInfoManager; + private BlockingQueue unRegisterQueue; + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); + + public BatchUnRegisterService(RouteInfoManager routeInfoManager, NamesrvConfig namesrvConfig) { + this.routeInfoManager = routeInfoManager; + this.unRegisterQueue = new LinkedBlockingQueue<>(namesrvConfig.getUnRegisterBrokerQueueCapacity()); + } + + /** + * Submits an unregister request to this queue. + * + * @param unRegisterRequest the request to submit + * @return {@code true} if the request was added to this queue, else {@code false} + */ + public boolean submit(UnRegisterBrokerRequestHeader unRegisterRequest) { + return unRegisterQueue.offer(unRegisterRequest); + } + + @Override + public String getServiceName() { + return BatchUnRegisterService.class.getName(); + } + + @Override + public void run() { + while (!this.isStopped()) { + try { + final UnRegisterBrokerRequestHeader request = unRegisterQueue.poll(3, TimeUnit.SECONDS); + if (request != null) { + Set unRegisterRequests = new HashSet<>(); + unRegisterQueue.drainTo(unRegisterRequests); + + // Add polled request + unRegisterRequests.add(request); + + this.routeInfoManager.unRegisterBroker(unRegisterRequests); + } + } catch (Throwable e) { + log.error("Handle unregister broker request failed", e); + } + } + } + + // For test only + int queueLength() { + return this.unRegisterQueue.size(); + } +} diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java index fedb4ae610..dbf4786151 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java @@ -37,16 +37,16 @@ public class BrokerHousekeepingService implements ChannelEventListener { @Override public void onChannelClose(String remoteAddr, Channel channel) { - this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + this.namesrvController.getRouteInfoManager().onChannelDestroy(channel); } @Override public void onChannelException(String remoteAddr, Channel channel) { - this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + this.namesrvController.getRouteInfoManager().onChannelDestroy(channel); } @Override public void onChannelIdle(String remoteAddr, Channel channel) { - this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + this.namesrvController.getRouteInfoManager().onChannelDestroy(channel); } } diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index a02d3f1f44..4becdd6ea9 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -16,26 +16,34 @@ */ package org.apache.rocketmq.namesrv.routeinfo; +import com.google.common.collect.Sets; import io.netty.channel.Channel; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.protocol.body.BrokerMemberGroup; +import org.apache.rocketmq.common.protocol.header.NotifyMinBrokerIdChangeRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; @@ -46,27 +54,61 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class RouteInfoManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); - private final static long BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2; + private final static long DEFAULT_BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2; private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private final HashMap> topicQueueTable; - private final HashMap brokerAddrTable; - private final HashMap> clusterAddrTable; - private final HashMap brokerLiveTable; - private final HashMap/* Filter Server */> filterServerTable; - private final HashMap> topicQueueMappingInfoTable; + private final Map> topicQueueTable; + private final Map brokerAddrTable; + private final Map> clusterAddrTable; + private final Map brokerLiveTable; + private final Map/* Filter Server */> filterServerTable; + private final Map> topicQueueMappingInfoTable; + // Contains topic route data needed by clients + private final Map topicRouteDataMap; - public RouteInfoManager() { - this.topicQueueTable = new HashMap>(1024); - this.brokerAddrTable = new HashMap(128); - this.clusterAddrTable = new HashMap>(32); - this.brokerLiveTable = new HashMap(256); - this.filterServerTable = new HashMap>(256); - this.topicQueueMappingInfoTable = new HashMap>(1024); + private final BatchUnRegisterService unRegisterService; + + private final NamesrvController namesrvController; + private final NamesrvConfig namesrvConfig; + + public RouteInfoManager(final NamesrvConfig namesrvConfig, NamesrvController namesrvController) { + this.topicQueueTable = new ConcurrentHashMap>(1024); + this.brokerAddrTable = new ConcurrentHashMap(128); + this.clusterAddrTable = new ConcurrentHashMap>(32); + this.brokerLiveTable = new ConcurrentHashMap(256); + this.filterServerTable = new ConcurrentHashMap>(256); + this.topicQueueMappingInfoTable = new ConcurrentHashMap>(1024); + this.topicRouteDataMap = new ConcurrentHashMap<>(1024); + this.unRegisterService = new BatchUnRegisterService(this, namesrvConfig); + this.namesrvConfig = namesrvConfig; + this.namesrvController = namesrvController; + } + + public void start() { + this.unRegisterService.start(); + } + + public void shutdown() { + this.unRegisterService.shutdown(true); + } + + public boolean submitUnRegisterBrokerRequest(UnRegisterBrokerRequestHeader unRegisterRequest) { + return this.unRegisterService.submit(unRegisterRequest); + } + + // For test only + int blockedUnRegisterRequests() { + return this.unRegisterService.queueLength(); } public byte[] getAllClusterInfo() { @@ -76,11 +118,76 @@ public class RouteInfoManager { return clusterInfoSerializeWrapper.encode(); } + public void registerTopic(final String topic, List queueDatas) { + if (queueDatas == null || queueDatas.isEmpty()) { + return; + } + try { + try { + this.lock.writeLock().lockInterruptibly(); + if (this.topicQueueTable.containsKey(topic)) { + log.info("Topic route already exist.{}, {}", topic, this.topicQueueTable.get(topic)); + } else { + // check and construct queue data map + Map queueDataMap = new HashMap<>(); + for (QueueData queueData : queueDatas) { + if (!this.brokerAddrTable.containsKey(queueData.getBrokerName())) { + log.warn("Register topic contains illegal broker, {}, {}", topic, queueData); + return; + } + queueDataMap.put(queueData.getBrokerName(), queueData); + } + + this.topicQueueTable.put(topic, queueDataMap); + log.info("Register topic route:{}, {}", topic, queueDatas); + + updateTopicRouteData(Sets.newHashSet(topic)); + } + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("registerTopic Exception", e); + } + } + public void deleteTopic(final String topic) { try { try { this.lock.writeLock().lockInterruptibly(); this.topicQueueTable.remove(topic); + this.updateTopicRouteData(Sets.newHashSet(topic)); + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("deleteTopic Exception", e); + } + } + + public void deleteTopic(final String topic, final String clusterName) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + Set brokerNames = this.clusterAddrTable.get(clusterName); + if (brokerNames != null + && !brokerNames.isEmpty()) { + Map queueDataMap = this.topicQueueTable.get(topic); + for (String brokerName : brokerNames) { + final QueueData removedQD = queueDataMap.remove(brokerName); + if (removedQD != null) { + log.info("deleteTopic, remove one broker's topic {} {} {}", brokerName, topic, + removedQD); + } + + if (queueDataMap.isEmpty()) { + log.info("deleteTopic, remove the topic all queue {} {}", brokerName, topic); + this.topicQueueTable.remove(topic); + break; + } + } + this.updateTopicRouteData(Sets.newHashSet(topic)); + } } finally { this.lock.writeLock().unlock(); } @@ -111,6 +218,21 @@ public class RouteInfoManager { final String brokerName, final long brokerId, final String haServerAddr, + final Long timeoutMillis, + final TopicConfigSerializeWrapper topicConfigWrapper, + final List filterServerList, + final Channel channel) { + return registerBroker(clusterName, brokerAddr, brokerName, brokerId, haServerAddr, timeoutMillis, false, topicConfigWrapper, filterServerList, channel); + } + + public RegisterBrokerResult registerBroker( + final String clusterName, + final String brokerAddr, + final String brokerName, + final long brokerId, + final String haServerAddr, + final Long timeoutMillis, + final Boolean enableActingMaster, final TopicConfigSerializeWrapper topicConfigWrapper, final List filterServerList, final Channel channel) { @@ -119,11 +241,7 @@ public class RouteInfoManager { try { this.lock.writeLock().lockInterruptibly(); - Set brokerNames = this.clusterAddrTable.get(clusterName); - if (null == brokerNames) { - brokerNames = new HashSet(); - this.clusterAddrTable.put(clusterName, brokerNames); - } + Set brokerNames = this.clusterAddrTable.computeIfAbsent(clusterName, k -> new HashSet<>()); brokerNames.add(brokerName); boolean registerFirst = false; @@ -131,37 +249,81 @@ public class RouteInfoManager { BrokerData brokerData = this.brokerAddrTable.get(brokerName); if (null == brokerData) { registerFirst = true; - brokerData = new BrokerData(clusterName, brokerName, new HashMap()); + brokerData = new BrokerData(clusterName, brokerName, new HashMap<>()); this.brokerAddrTable.put(brokerName, brokerData); } + + boolean isOldVersionBroker = enableActingMaster == null; + brokerData.setEnableActingMaster(!isOldVersionBroker && enableActingMaster); + Map brokerAddrsMap = brokerData.getBrokerAddrs(); + + boolean isMinBrokerIdChanged = false; + long prevMinBrokerId = 0; + if (!brokerAddrsMap.isEmpty()) { + prevMinBrokerId = Collections.min(brokerAddrsMap.keySet()); + } + + if (brokerId < prevMinBrokerId) { + isMinBrokerIdChanged = true; + } + //Switch slave to master: first remove <1, IP:PORT> in namesrv, then add <0, IP:PORT> //The same IP:PORT must only have one record in brokerAddrTable - Iterator> it = brokerAddrsMap.entrySet().iterator(); - while (it.hasNext()) { - Entry item = it.next(); - if (null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey()) { - it.remove(); + brokerAddrsMap.entrySet().removeIf(item -> null != brokerAddr && brokerAddr.equals(item.getValue()) && brokerId != item.getKey()); + + //If Local brokerId stateVersion bigger than the registering one, + String oldBrokerAddr = brokerAddrsMap.get(brokerId); + if (null != oldBrokerAddr && !oldBrokerAddr.equals(brokerAddr)) { + BrokerLiveInfo oldBrokerInfo = brokerLiveTable.get(new BrokerAddrInfo(clusterName, oldBrokerAddr)); + + if (null != oldBrokerInfo) { + long oldStateVersion = oldBrokerInfo.getDataVersion().getStateVersion(); + long newStateVersion = topicConfigWrapper.getDataVersion().getStateVersion(); + if (oldStateVersion > newStateVersion) { + log.warn("Registered Broker conflicts with the existed one, just ignore.: Cluster:{}, BrokerName:{}, BrokerId:{}, " + + "Old BrokerAddr:{}, Old Version:{}, New BrokerAddr:{}, New Version:{}.", + clusterName, brokerName, brokerId, oldBrokerAddr, oldStateVersion, brokerAddr, newStateVersion); + //Remove the rejected brokerAddr from brokerLiveTable. + brokerLiveTable.remove(new BrokerAddrInfo(clusterName, brokerAddr)); + return result; + } } } - String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr); + String oldAddr = brokerAddrsMap.put(brokerId, brokerAddr); registerFirst = registerFirst || (null == oldAddr); - if (null != topicConfigWrapper - && MixAll.MASTER_ID == brokerId) { - if (this.isBrokerTopicConfigChanged(brokerAddr, topicConfigWrapper.getDataVersion()) - || registerFirst) { - ConcurrentMap tcTable = - topicConfigWrapper.getTopicConfigTable(); - if (tcTable != null) { - for (Map.Entry entry : tcTable.entrySet()) { - this.createAndUpdateQueueData(brokerName, entry.getValue()); + Set configChangedTopics = new HashSet<>(); + + boolean isMaster = MixAll.MASTER_ID == brokerId; + boolean isPrimeSlave = !isOldVersionBroker && !isMaster + && brokerId == Collections.min(brokerAddrsMap.keySet()); + + if (null != topicConfigWrapper && (isMaster || isPrimeSlave)) { + + ConcurrentMap tcTable = + topicConfigWrapper.getTopicConfigTable(); + if (tcTable != null) { + for (Map.Entry entry : tcTable.entrySet()) { + final TopicConfig topicConfig = entry.getValue(); + String topicName = topicConfig.getTopicName(); + if (registerFirst || this.isTopicConfigChanged(clusterName, brokerAddr, + topicConfigWrapper.getDataVersion(), brokerName, topicName)) { + configChangedTopics.add(topicName); + + if (isPrimeSlave) { + // Wipe the write perm for prime slave + topicConfig.setPerm(topicConfig.getPerm() & (~PermName.PERM_WRITE)); + } + + this.createAndUpdateQueueData(brokerName, topicConfig); } } + } + if (this.isBrokerTopicConfigChanged(clusterName, brokerAddr, topicConfigWrapper.getDataVersion()) || registerFirst) { TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper = TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper); - Map topicQueueMappingInfoMap = mappingSerializeWrapper.getTopicQueueMappingInfoMap(); //the topicQueueMappingInfoMap should never be null, but can be empty for (Map.Entry entry : topicQueueMappingInfoMap.entrySet()) { @@ -175,34 +337,49 @@ public class RouteInfoManager { } } - BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddr, + BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(clusterName, brokerAddr); + BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddrInfo, new BrokerLiveInfo( System.currentTimeMillis(), - topicConfigWrapper != null ? topicConfigWrapper.getDataVersion() : new DataVersion(), + timeoutMillis == null ? DEFAULT_BROKER_CHANNEL_EXPIRED_TIME : timeoutMillis, + topicConfigWrapper == null ? new DataVersion() : topicConfigWrapper.getDataVersion(), channel, haServerAddr)); if (null == prevBrokerLiveInfo) { - log.info("new broker registered, {} HAServer: {}", brokerAddr, haServerAddr); + log.info("new broker registered, {} HAService: {}", brokerAddrInfo, haServerAddr); } if (filterServerList != null) { if (filterServerList.isEmpty()) { - this.filterServerTable.remove(brokerAddr); + this.filterServerTable.remove(brokerAddrInfo); } else { - this.filterServerTable.put(brokerAddr, filterServerList); + this.filterServerTable.put(brokerAddrInfo, filterServerList); } } if (MixAll.MASTER_ID != brokerId) { String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); if (masterAddr != null) { - BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.get(masterAddr); - if (brokerLiveInfo != null) { - result.setHaServerAddr(brokerLiveInfo.getHaServerAddr()); + BrokerAddrInfo masterAddrInfo = new BrokerAddrInfo(clusterName, masterAddr); + BrokerLiveInfo masterLiveInfo = this.brokerLiveTable.get(masterAddrInfo); + if (masterLiveInfo != null) { + result.setHaServerAddr(masterLiveInfo.getHaServerAddr()); result.setMasterAddr(masterAddr); } } } + + if (registerFirst && MixAll.MASTER_ID != brokerId) { + configChangedTopics = this.topicSetOfBrokerName(brokerName); + } + + // Update topicRouteDataMap + updateTopicRouteData(configChangedTopics); + + if (isMinBrokerIdChanged && brokerData.isEnableActingMaster()) { + notifyMinBrokerIdChanged(brokerAddrsMap, null, + this.brokerLiveTable.get(brokerAddrInfo).getHaServerAddr()); + } } finally { this.lock.writeLock().unlock(); } @@ -213,21 +390,109 @@ public class RouteInfoManager { return result; } - public boolean isBrokerTopicConfigChanged(final String brokerAddr, final DataVersion dataVersion) { - DataVersion prev = queryBrokerTopicConfig(brokerAddr); + private void updateTopicRouteData(final Set changedTopics) { + for (final String changedTopic : changedTopics) { + if (!this.topicQueueTable.containsKey(changedTopic)) { + // This topic doesn't have any queue data route info + // Just remove it from topicRouteDataMap + this.topicRouteDataMap.remove(changedTopic); + continue; + } + + TopicRouteData topicRouteData = new TopicRouteData(); + boolean foundBrokerData = false; + List brokerDataList = topicRouteData.getBrokerDatas(); + HashMap> filterServerMap = topicRouteData.getFilterServerTable(); + + Map queueDataMap = this.topicQueueTable.get(changedTopic); + if (queueDataMap != null) { + // Deep copy the queue data list + List queueDataList = topicRouteData.getQueueDatas(); + for (final QueueData queueData : queueDataMap.values()) { + queueDataList.add(new QueueData(queueData)); + } + + Set brokerNameSet = new HashSet(queueDataMap.keySet()); + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + BrokerData brokerDataClone = new BrokerData(brokerData); + brokerDataList.add(brokerDataClone); + foundBrokerData = true; + + for (final String brokerAddr : brokerDataClone.getBrokerAddrs().values()) { + BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(brokerDataClone.getCluster(), brokerAddr); + List filterServerList = this.filterServerTable.get(brokerAddrInfo); + if (filterServerList != null) { + filterServerMap.put(brokerAddr, new ArrayList(filterServerList)); + } + } + } + } + } + + if (foundBrokerData) { + this.topicRouteDataMap.put(changedTopic, topicRouteData); + } + } + } + + public BrokerMemberGroup getBrokerMemberGroup(String clusterName, String brokerName) { + BrokerMemberGroup groupMember = new BrokerMemberGroup(clusterName, brokerName); + try { + try { + this.lock.readLock().lockInterruptibly(); + final BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (brokerData != null) { + groupMember.getBrokerAddrs().putAll(brokerData.getBrokerAddrs()); + } + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("Get broker member group exception", e); + } + return groupMember; + } + + public boolean isBrokerTopicConfigChanged(final String clusterName, final String brokerAddr, + final DataVersion dataVersion) { + DataVersion prev = queryBrokerTopicConfig(clusterName, brokerAddr); return null == prev || !prev.equals(dataVersion); } - public DataVersion queryBrokerTopicConfig(final String brokerAddr) { - BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); + public boolean isTopicConfigChanged(final String clusterName, final String brokerAddr, + final DataVersion dataVersion, String brokerName, String topic) { + boolean isChange = isBrokerTopicConfigChanged(clusterName, brokerAddr, dataVersion); + if (isChange) { + return true; + } + final Map queueDataMap = this.topicQueueTable.get(topic); + if (queueDataMap == null || queueDataMap.isEmpty()) { + return true; + } + + if (queueDataMap.containsKey(brokerName)) { + // The topicQueueTable already contains the broker + return false; + } + + return true; + } + + public DataVersion queryBrokerTopicConfig(final String clusterName, final String brokerAddr) { + BrokerAddrInfo addrInfo = new BrokerAddrInfo(clusterName, brokerAddr); + BrokerLiveInfo prev = this.brokerLiveTable.get(addrInfo); if (prev != null) { return prev.getDataVersion(); } return null; } - public void updateBrokerInfoUpdateTimestamp(final String brokerAddr) { - BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); + public void updateBrokerInfoUpdateTimestamp(final String clusterName, final String brokerAddr) { + BrokerAddrInfo addrInfo = new BrokerAddrInfo(clusterName, brokerAddr); + BrokerLiveInfo prev = this.brokerLiveTable.get(addrInfo); if (prev != null) { prev.setLastUpdateTimestamp(System.currentTimeMillis()); } @@ -241,81 +506,79 @@ public class RouteInfoManager { queueData.setPerm(topicConfig.getPerm()); queueData.setTopicSysFlag(topicConfig.getTopicSysFlag()); - List queueDataList = this.topicQueueTable.get(topicConfig.getTopicName()); - if (null == queueDataList) { - queueDataList = new LinkedList(); - queueDataList.add(queueData); - this.topicQueueTable.put(topicConfig.getTopicName(), queueDataList); + Map queueDataMap = this.topicQueueTable.get(topicConfig.getTopicName()); + if (null == queueDataMap) { + queueDataMap = new HashMap<>(); + queueDataMap.put(brokerName, queueData); + this.topicQueueTable.put(topicConfig.getTopicName(), queueDataMap); log.info("new topic registered, {} {}", topicConfig.getTopicName(), queueData); } else { - boolean addNewOne = true; - - Iterator it = queueDataList.iterator(); - while (it.hasNext()) { - QueueData qd = it.next(); - if (qd.getBrokerName().equals(brokerName)) { - if (qd.equals(queueData)) { - addNewOne = false; - } else { - log.info("topic changed, {} OLD: {} NEW: {}", topicConfig.getTopicName(), qd, - queueData); - it.remove(); - } - } - } - - if (addNewOne) { - queueDataList.add(queueData); + final QueueData existedQD = queueDataMap.get(brokerName); + if (existedQD == null) { + queueDataMap.put(brokerName, queueData); + } else if (!existedQD.equals(queueData)) { + log.info("topic changed, {} OLD: {} NEW: {}", topicConfig.getTopicName(), existedQD, + queueData); + queueDataMap.put(brokerName, queueData); } } } public int wipeWritePermOfBrokerByLock(final String brokerName) { - return operateWritePermOfBrokerByLock(brokerName, RequestCode.WIPE_WRITE_PERM_OF_BROKER); - } - - public int addWritePermOfBrokerByLock(final String brokerName) { - return operateWritePermOfBrokerByLock(brokerName, RequestCode.ADD_WRITE_PERM_OF_BROKER); - } - - private int operateWritePermOfBrokerByLock(final String brokerName, final int requestCode) { try { try { this.lock.writeLock().lockInterruptibly(); - return operateWritePermOfBroker(brokerName, requestCode); + return operateWritePermOfBroker(brokerName, RequestCode.WIPE_WRITE_PERM_OF_BROKER); } finally { this.lock.writeLock().unlock(); } } catch (Exception e) { - log.error("operateWritePermOfBrokerByLock Exception", e); + log.error("wipeWritePermOfBrokerByLock Exception", e); } return 0; } + public int addWritePermOfBrokerByLock(final String brokerName) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + return operateWritePermOfBroker(brokerName, RequestCode.ADD_WRITE_PERM_OF_BROKER); + } finally { + this.lock.writeLock().unlock(); + } + } catch (Exception e) { + log.error("wipeWritePermOfBrokerByLock Exception", e); + } + return 0; + } private int operateWritePermOfBroker(final String brokerName, final int requestCode) { + Set changedTopics = new HashSet<>(); int topicCnt = 0; - for (Entry> entry : this.topicQueueTable.entrySet()) { - List qdList = entry.getValue(); - for (QueueData qd : qdList) { - if (qd.getBrokerName().equals(brokerName)) { - int perm = qd.getPerm(); - switch (requestCode) { - case RequestCode.WIPE_WRITE_PERM_OF_BROKER: - perm &= ~PermName.PERM_WRITE; - break; - case RequestCode.ADD_WRITE_PERM_OF_BROKER: - perm = PermName.PERM_READ | PermName.PERM_WRITE; - break; - } - qd.setPerm(perm); - topicCnt++; + Iterator>> itTopic = this.topicQueueTable.entrySet().iterator(); + while (itTopic.hasNext()) { + Entry> entry = itTopic.next(); + Map qdMap = entry.getValue(); + + final QueueData qd = qdMap.get(brokerName); + if (qd != null) { + changedTopics.add(entry.getKey()); + int perm = qd.getPerm(); + switch (requestCode) { + case RequestCode.WIPE_WRITE_PERM_OF_BROKER: + perm &= ~PermName.PERM_WRITE; + break; + case RequestCode.ADD_WRITE_PERM_OF_BROKER: + perm = PermName.PERM_READ | PermName.PERM_WRITE; + break; } + qd.setPerm(perm); + topicCnt++; } } - + this.updateTopicRouteData(changedTopics); return topicCnt; } @@ -324,52 +587,89 @@ public class RouteInfoManager { final String brokerAddr, final String brokerName, final long brokerId) { + UnRegisterBrokerRequestHeader unRegisterBrokerRequest = new UnRegisterBrokerRequestHeader(); + unRegisterBrokerRequest.setClusterName(clusterName); + unRegisterBrokerRequest.setBrokerAddr(brokerAddr); + unRegisterBrokerRequest.setBrokerName(brokerName); + unRegisterBrokerRequest.setBrokerId(brokerId); + + unRegisterBroker(Sets.newHashSet(unRegisterBrokerRequest)); + } + + public void unRegisterBroker(Set unRegisterRequests) { try { try { + Set removedBroker = new HashSet<>(); + Set reducedBroker = new HashSet<>(); + Map needNotifyBrokerMap = new HashMap<>(); + this.lock.writeLock().lockInterruptibly(); - BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddr); - log.info("unregisterBroker, remove from brokerLiveTable {}, {}", - brokerLiveInfo != null ? "OK" : "Failed", - brokerAddr - ); + for (final UnRegisterBrokerRequestHeader unRegisterRequest : unRegisterRequests) { + final String brokerName = unRegisterRequest.getBrokerName(); + final String clusterName = unRegisterRequest.getClusterName(); - this.filterServerTable.remove(brokerAddr); + BrokerAddrInfo brokerAddrInfo = new BrokerAddrInfo(clusterName, unRegisterRequest.getBrokerAddr()); - boolean removeBrokerName = false; - BrokerData brokerData = this.brokerAddrTable.get(brokerName); - if (null != brokerData) { - String addr = brokerData.getBrokerAddrs().remove(brokerId); - log.info("unregisterBroker, remove addr from brokerAddrTable {}, {}", - addr != null ? "OK" : "Failed", - brokerAddr + BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddrInfo); + log.info("unregisterBroker, remove from brokerLiveTable {}, {}", + brokerLiveInfo != null ? "OK" : "Failed", + brokerAddrInfo ); - if (brokerData.getBrokerAddrs().isEmpty()) { - this.brokerAddrTable.remove(brokerName); - log.info("unregisterBroker, remove name from brokerAddrTable OK, {}", - brokerName - ); + this.filterServerTable.remove(brokerAddrInfo); - removeBrokerName = true; + boolean removeBrokerName = false; + boolean isMinBrokerIdChanged = false; + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + if (!brokerData.getBrokerAddrs().isEmpty() && + unRegisterRequest.getBrokerId().equals(Collections.min(brokerData.getBrokerAddrs().keySet()))) { + isMinBrokerIdChanged = true; + } + String addr = brokerData.getBrokerAddrs().remove(unRegisterRequest.getBrokerId()); + log.info("unregisterBroker, remove addr from brokerAddrTable {}, {}", + addr != null ? "OK" : "Failed", + brokerAddrInfo + ); + if (brokerData.getBrokerAddrs().isEmpty()) { + this.brokerAddrTable.remove(brokerName); + log.info("unregisterBroker, remove name from brokerAddrTable OK, {}", + brokerName + ); + + removeBrokerName = true; + } else if (isMinBrokerIdChanged) { + needNotifyBrokerMap.put(brokerName, new BrokerStatusChangeInfo( + brokerData.getBrokerAddrs(), addr, null)); + } + } + + if (removeBrokerName) { + Set nameSet = this.clusterAddrTable.get(clusterName); + if (nameSet != null) { + boolean removed = nameSet.remove(brokerName); + log.info("unregisterBroker, remove name from clusterAddrTable {}, {}", + removed ? "OK" : "Failed", + brokerName); + + if (nameSet.isEmpty()) { + this.clusterAddrTable.remove(clusterName); + log.info("unregisterBroker, remove cluster from clusterAddrTable {}", + clusterName + ); + } + } + removedBroker.add(brokerName); + } else { + reducedBroker.add(brokerName); } } - if (removeBrokerName) { - Set nameSet = this.clusterAddrTable.get(clusterName); - if (nameSet != null) { - boolean removed = nameSet.remove(brokerName); - log.info("unregisterBroker, remove name from clusterAddrTable {}, {}", - removed ? "OK" : "Failed", - brokerName); + Set changedTopics = cleanTopicByUnRegisterRequests(removedBroker, reducedBroker); + this.updateTopicRouteData(changedTopics); - if (nameSet.isEmpty()) { - this.clusterAddrTable.remove(clusterName); - log.info("unregisterBroker, remove cluster from clusterAddrTable {}", - clusterName - ); - } - } - this.removeTopicByBrokerName(brokerName); + if (!needNotifyBrokerMap.isEmpty()) { + notifyMinBrokerIdChanged(needNotifyBrokerMap); } } finally { this.lock.writeLock().unlock(); @@ -379,114 +679,189 @@ public class RouteInfoManager { } } - private void removeTopicByBrokerName(final String brokerName) { - Iterator>> itMap = this.topicQueueTable.entrySet().iterator(); + private Set cleanTopicByUnRegisterRequests(Set removedBroker, Set reducedBroker) { + Set changedTopics = new HashSet<>(); + Iterator>> itMap = this.topicQueueTable.entrySet().iterator(); while (itMap.hasNext()) { - Entry> entry = itMap.next(); + Entry> entry = itMap.next(); String topic = entry.getKey(); - List queueDataList = entry.getValue(); - Iterator it = queueDataList.iterator(); - while (it.hasNext()) { - QueueData qd = it.next(); - if (qd.getBrokerName().equals(brokerName)) { - log.info("removeTopicByBrokerName, remove one broker's topic {} {}", topic, qd); - it.remove(); + Map queueDataMap = entry.getValue(); + + for (final String brokerName : removedBroker) { + final QueueData removedQD = queueDataMap.remove(brokerName); + if (removedQD != null) { + changedTopics.add(topic); + log.info("removeTopicByBrokerName, remove one broker's topic {} {}", topic, removedQD); } } - if (queueDataList.isEmpty()) { + if (queueDataMap.isEmpty()) { log.info("removeTopicByBrokerName, remove the topic all queue {}", topic); itMap.remove(); } - } - } - public TopicRouteData pickupTopicRouteData(final String topic) { - TopicRouteData topicRouteData = new TopicRouteData(); - boolean foundQueueData = false; - boolean foundBrokerData = false; - Set brokerNameSet = new HashSet(); - List brokerDataList = new LinkedList(); - topicRouteData.setBrokerDatas(brokerDataList); + for (final String brokerName : reducedBroker) { + final QueueData queueData = queueDataMap.get(brokerName); - HashMap> filterServerMap = new HashMap>(); - topicRouteData.setFilterServerTable(filterServerMap); - topicRouteData.setTopicQueueMappingByBroker(this.topicQueueMappingInfoTable.get(topic)); - - try { - try { - this.lock.readLock().lockInterruptibly(); - List queueDataList = this.topicQueueTable.get(topic); - if (queueDataList != null) { - topicRouteData.setQueueDatas(queueDataList); - foundQueueData = true; - - Iterator it = queueDataList.iterator(); - while (it.hasNext()) { - QueueData qd = it.next(); - brokerNameSet.add(qd.getBrokerName()); - } - - for (String brokerName : brokerNameSet) { - BrokerData brokerData = this.brokerAddrTable.get(brokerName); - if (null != brokerData) { - BrokerData brokerDataClone = new BrokerData(brokerData.getCluster(), brokerData.getBrokerName(), (HashMap) brokerData - .getBrokerAddrs().clone()); - brokerDataList.add(brokerDataClone); - foundBrokerData = true; - for (final String brokerAddr : brokerDataClone.getBrokerAddrs().values()) { - List filterServerList = this.filterServerTable.get(brokerAddr); - filterServerMap.put(brokerAddr, filterServerList); - } + if (queueData != null) { + changedTopics.add(topic); + if (this.brokerAddrTable.get(brokerName).isEnableActingMaster()) { + // Master has been unregistered, wipe the write perm + if (isNoMasterExists(brokerName)) { + queueData.setPerm(queueData.getPerm() & (~PermName.PERM_WRITE)); } } } - } finally { - this.lock.readLock().unlock(); } - } catch (Exception e) { - log.error("pickupTopicRouteData Exception", e); } - log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); + return changedTopics; + } - if (foundBrokerData && foundQueueData) { - return topicRouteData; + private boolean isNoMasterExists(String brokerName) { + final BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (brokerData == null) { + return true; } + if (brokerData.getBrokerAddrs().size() == 0) { + return true; + } + + return Collections.min(brokerData.getBrokerAddrs().keySet()) > 0; + } + + private Set topicSetOfBrokerName(final String brokerName) { + Set topicOfBroker = new HashSet<>(); + for (final Entry> entry : this.topicQueueTable.entrySet()) { + if (entry.getValue().containsKey(brokerName)) { + topicOfBroker.add(entry.getKey()); + } + } + return topicOfBroker; + } + + public TopicRouteData pickupTopicRouteData(final String topic) { + if (topic != null) { + final TopicRouteData routeData = this.topicRouteDataMap.get(topic); + if (routeData == null) { + return null; + } + routeData.setTopicQueueMappingByBroker(this.topicQueueMappingInfoTable.get(topic)); + log.debug("pickupTopicRouteData {} {}", topic, routeData); + + if (topic.startsWith(TopicValidator.SYNC_BROKER_MEMBER_GROUP_PREFIX)) { + return routeData; + } + + if (!namesrvConfig.isSupportActingMaster()) { + return routeData; + } + + if (routeData.getBrokerDatas().size() == 0 || routeData.getQueueDatas().size() == 0) { + return routeData; + } + + boolean needActingMaster = false; + + for (final BrokerData brokerData : routeData.getBrokerDatas()) { + if (brokerData.getBrokerAddrs().size() != 0 + && !brokerData.getBrokerAddrs().containsKey(MixAll.MASTER_ID)) { + needActingMaster = true; + break; + } + } + + if (!needActingMaster) { + return routeData; + } + + final TopicRouteData cloneTopicRouteData = routeData.deepCloneTopicRouteData(); + + for (final BrokerData brokerData : cloneTopicRouteData.getBrokerDatas()) { + final HashMap brokerAddrs = brokerData.getBrokerAddrs(); + if (brokerAddrs.size() == 0 || brokerAddrs.containsKey(MixAll.MASTER_ID) || !brokerData.isEnableActingMaster()) { + continue; + } + + // No master + for (final QueueData queueData : cloneTopicRouteData.getQueueDatas()) { + if (queueData.getBrokerName().equals(brokerData.getBrokerName())) { + if (!PermName.isWriteable(queueData.getPerm())) { + final Long minBrokerId = Collections.min(brokerAddrs.keySet()); + final String actingMasterAddr = brokerAddrs.remove(minBrokerId); + brokerAddrs.put(MixAll.MASTER_ID, actingMasterAddr); + } + break; + } + } + + } + + return cloneTopicRouteData; + } return null; } public void scanNotActiveBroker() { - Iterator> it = this.brokerLiveTable.entrySet().iterator(); - while (it.hasNext()) { - Entry next = it.next(); - long last = next.getValue().getLastUpdateTimestamp(); - if ((last + BROKER_CHANNEL_EXPIRED_TIME) < System.currentTimeMillis()) { - RemotingUtil.closeChannel(next.getValue().getChannel()); - it.remove(); - log.warn("The broker channel expired, {} {}ms", next.getKey(), BROKER_CHANNEL_EXPIRED_TIME); - this.onChannelDestroy(next.getKey(), next.getValue().getChannel()); + try { + log.info("start scanNotActiveBroker"); + for (Entry next : this.brokerLiveTable.entrySet()) { + long last = next.getValue().getLastUpdateTimestamp(); + long timeoutMillis = next.getValue().getHeartbeatTimeoutMillis(); + if ((last + timeoutMillis) < System.currentTimeMillis()) { + RemotingUtil.closeChannel(next.getValue().getChannel()); + log.warn("The broker channel expired, {} {}ms", next.getKey(), timeoutMillis); + this.onChannelDestroy(next.getKey()); + } } + } catch (Exception e) { + log.error("scanNotActiveBroker exception", e); } } - public void onChannelDestroy(String remoteAddr, Channel channel) { - String brokerAddrFound = null; + public void onChannelDestroy(BrokerAddrInfo brokerAddrInfo) { + UnRegisterBrokerRequestHeader unRegisterRequest = new UnRegisterBrokerRequestHeader(); + boolean needUnRegister = false; + if (brokerAddrInfo != null) { + try { + try { + this.lock.readLock().lockInterruptibly(); + needUnRegister = setupUnRegisterRequest(unRegisterRequest, brokerAddrInfo); + } finally { + this.lock.readLock().unlock(); + } + } catch (Exception e) { + log.error("onChannelDestroy Exception", e); + } + } + + if (needUnRegister) { + boolean result = this.submitUnRegisterBrokerRequest(unRegisterRequest); + log.info("the broker's channel destroyed, submit the unregister request at once, " + + "broker info: {}, submit result: {}", unRegisterRequest, result); + } + } + + public void onChannelDestroy(Channel channel) { + UnRegisterBrokerRequestHeader unRegisterRequest = new UnRegisterBrokerRequestHeader(); + BrokerAddrInfo brokerAddrFound = null; + boolean needUnRegister = false; if (channel != null) { try { try { this.lock.readLock().lockInterruptibly(); - Iterator> itBrokerLiveTable = - this.brokerLiveTable.entrySet().iterator(); - while (itBrokerLiveTable.hasNext()) { - Entry entry = itBrokerLiveTable.next(); + for (Entry entry : this.brokerLiveTable.entrySet()) { if (entry.getValue().getChannel() == channel) { brokerAddrFound = entry.getKey(); break; } } + + if (brokerAddrFound != null) { + needUnRegister = setupUnRegisterRequest(unRegisterRequest, brokerAddrFound); + } } finally { this.lock.readLock().unlock(); } @@ -495,104 +870,93 @@ public class RouteInfoManager { } } - if (null == brokerAddrFound) { - brokerAddrFound = remoteAddr; - } else { - log.info("the broker's channel destroyed, {}, clean it's data structure at once", brokerAddrFound); + if (needUnRegister) { + boolean result = this.submitUnRegisterBrokerRequest(unRegisterRequest); + log.info("the broker's channel destroyed, submit the unregister request at once, " + + "broker info: {}, submit result: {}", unRegisterRequest, result); + } + } + + private boolean setupUnRegisterRequest(UnRegisterBrokerRequestHeader unRegisterRequest, + BrokerAddrInfo brokerAddrInfo) { + unRegisterRequest.setClusterName(brokerAddrInfo.getClusterName()); + unRegisterRequest.setBrokerAddr(brokerAddrInfo.getBrokerAddr()); + + for (Entry stringBrokerDataEntry : this.brokerAddrTable.entrySet()) { + BrokerData brokerData = stringBrokerDataEntry.getValue(); + if (!brokerAddrInfo.getClusterName().equals(brokerData.getCluster())) { + continue; + } + + for (Entry entry : brokerData.getBrokerAddrs().entrySet()) { + Long brokerId = entry.getKey(); + String brokerAddr = entry.getValue(); + if (brokerAddr.equals(brokerAddrInfo.getBrokerAddr())) { + unRegisterRequest.setBrokerName(brokerData.getBrokerName()); + unRegisterRequest.setBrokerId(brokerId); + return true; + } + } } - if (brokerAddrFound != null && brokerAddrFound.length() > 0) { + return false; + } - try { - try { - this.lock.writeLock().lockInterruptibly(); - this.brokerLiveTable.remove(brokerAddrFound); - this.filterServerTable.remove(brokerAddrFound); - String brokerNameFound = null; - boolean removeBrokerName = false; - Iterator> itBrokerAddrTable = - this.brokerAddrTable.entrySet().iterator(); - while (itBrokerAddrTable.hasNext() && (null == brokerNameFound)) { - BrokerData brokerData = itBrokerAddrTable.next().getValue(); - - Iterator> it = brokerData.getBrokerAddrs().entrySet().iterator(); - while (it.hasNext()) { - Entry entry = it.next(); - Long brokerId = entry.getKey(); - String brokerAddr = entry.getValue(); - if (brokerAddr.equals(brokerAddrFound)) { - brokerNameFound = brokerData.getBrokerName(); - it.remove(); - log.info("remove brokerAddr[{}, {}] from brokerAddrTable, because channel destroyed", - brokerId, brokerAddr); - break; - } - } - - if (brokerData.getBrokerAddrs().isEmpty()) { - removeBrokerName = true; - itBrokerAddrTable.remove(); - log.info("remove brokerName[{}] from brokerAddrTable, because channel destroyed", - brokerData.getBrokerName()); - } - } - - if (brokerNameFound != null && removeBrokerName) { - Iterator>> it = this.clusterAddrTable.entrySet().iterator(); - while (it.hasNext()) { - Entry> entry = it.next(); - String clusterName = entry.getKey(); - Set brokerNames = entry.getValue(); - boolean removed = brokerNames.remove(brokerNameFound); - if (removed) { - log.info("remove brokerName[{}], clusterName[{}] from clusterAddrTable, because channel destroyed", - brokerNameFound, clusterName); - - if (brokerNames.isEmpty()) { - log.info("remove the clusterName[{}] from clusterAddrTable, because channel destroyed and no broker in this cluster", - clusterName); - it.remove(); - } - - break; - } - } - } - - if (removeBrokerName) { - Iterator>> itTopicQueueTable = - this.topicQueueTable.entrySet().iterator(); - while (itTopicQueueTable.hasNext()) { - Entry> entry = itTopicQueueTable.next(); - String topic = entry.getKey(); - List queueDataList = entry.getValue(); - - Iterator itQueueData = queueDataList.iterator(); - while (itQueueData.hasNext()) { - QueueData queueData = itQueueData.next(); - if (queueData.getBrokerName().equals(brokerNameFound)) { - itQueueData.remove(); - log.info("remove topic[{} {}], from topicQueueTable, because channel destroyed", - topic, queueData); - } - } - - if (queueDataList.isEmpty()) { - itTopicQueueTable.remove(); - log.info("remove topic[{}] all queue, from topicQueueTable, because channel destroyed", - topic); - } - } - } - } finally { - this.lock.writeLock().unlock(); - } - } catch (Exception e) { - log.error("onChannelDestroy Exception", e); + private void notifyMinBrokerIdChanged(Map needNotifyBrokerMap) + throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, + RemotingTooMuchRequestException { + for (String brokerName : needNotifyBrokerMap.keySet()) { + BrokerStatusChangeInfo brokerStatusChangeInfo = needNotifyBrokerMap.get(brokerName); + BrokerData brokerData = brokerAddrTable.get(brokerName); + if (brokerData != null && brokerData.isEnableActingMaster()) { + notifyMinBrokerIdChanged(brokerStatusChangeInfo.getBrokerAddrs(), + brokerStatusChangeInfo.getOfflineBrokerAddr(), brokerStatusChangeInfo.getHaBrokerAddr()); } } } + private void notifyMinBrokerIdChanged(Map brokerAddrMap, String offlineBrokerAddr, + String haBrokerAddr) + throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, + RemotingTooMuchRequestException, RemotingConnectException { + if (brokerAddrMap == null || brokerAddrMap.isEmpty() || this.namesrvController == null) { + return; + } + + NotifyMinBrokerIdChangeRequestHeader requestHeader = new NotifyMinBrokerIdChangeRequestHeader(); + long minBrokerId = Collections.min(brokerAddrMap.keySet()); + requestHeader.setMinBrokerId(minBrokerId); + requestHeader.setMinBrokerAddr(brokerAddrMap.get(minBrokerId)); + requestHeader.setOfflineBrokerAddr(offlineBrokerAddr); + requestHeader.setHaBrokerAddr(haBrokerAddr); + + List brokerAddrsNotify = chooseBrokerAddrsToNotify(brokerAddrMap, offlineBrokerAddr); + log.info("min broker id changed to {}, notify {}, offline broker addr {}", minBrokerId, brokerAddrsNotify, offlineBrokerAddr); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.NOTIFY_MIN_BROKER_ID_CHANGE, requestHeader); + for (String brokerAddr : brokerAddrsNotify) { + this.namesrvController.getRemotingClient().invokeOneway(brokerAddr, request, 300); + } + } + + private List chooseBrokerAddrsToNotify(Map brokerAddrMap, String offlineBrokerAddr) { + if (offlineBrokerAddr != null || brokerAddrMap.size() == 1) { + // notify the reset brokers. + return new ArrayList<>(brokerAddrMap.values()); + } + + // new broker registered, notify previous brokers. + long minBrokerId = Collections.min(brokerAddrMap.keySet()); + List brokerAddrList = new ArrayList<>(); + for (Long brokerId : brokerAddrMap.keySet()) { + if (brokerId != minBrokerId) { + brokerAddrList.add(brokerAddrMap.get(brokerId)); + } + } + return brokerAddrList; + } + + // For test only public void printAllPeriodically() { try { try { @@ -600,9 +964,9 @@ public class RouteInfoManager { log.info("--------------------------------------------------------"); { log.info("topicQueueTable SIZE: {}", this.topicQueueTable.size()); - Iterator>> it = this.topicQueueTable.entrySet().iterator(); + Iterator>> it = this.topicQueueTable.entrySet().iterator(); while (it.hasNext()) { - Entry> next = it.next(); + Entry> next = it.next(); log.info("topicQueueTable Topic: {} {}", next.getKey(), next.getValue()); } } @@ -618,9 +982,9 @@ public class RouteInfoManager { { log.info("brokerLiveTable SIZE: {}", this.brokerLiveTable.size()); - Iterator> it = this.brokerLiveTable.entrySet().iterator(); + Iterator> it = this.brokerLiveTable.entrySet().iterator(); while (it.hasNext()) { - Entry next = it.next(); + Entry next = it.next(); log.info("brokerLiveTable brokerAddr: {} {}", next.getKey(), next.getValue()); } } @@ -680,17 +1044,15 @@ public class RouteInfoManager { this.lock.readLock().lockInterruptibly(); Set brokerNameSet = this.clusterAddrTable.get(cluster); for (String brokerName : brokerNameSet) { - Iterator>> topicTableIt = + Iterator>> topicTableIt = this.topicQueueTable.entrySet().iterator(); while (topicTableIt.hasNext()) { - Entry> topicEntry = topicTableIt.next(); + Entry> topicEntry = topicTableIt.next(); String topic = topicEntry.getKey(); - List queueDatas = topicEntry.getValue(); - for (QueueData queueData : queueDatas) { - if (brokerName.equals(queueData.getBrokerName())) { - topicList.getTopicList().add(topic); - break; - } + Map queueDataMap = topicEntry.getValue(); + final QueueData qd = queueDataMap.get(brokerName); + if (qd != null) { + topicList.getTopicList().add(topic); } } } @@ -709,14 +1071,14 @@ public class RouteInfoManager { try { try { this.lock.readLock().lockInterruptibly(); - Iterator>> topicTableIt = + Iterator>> topicTableIt = this.topicQueueTable.entrySet().iterator(); while (topicTableIt.hasNext()) { - Entry> topicEntry = topicTableIt.next(); + Entry> topicEntry = topicTableIt.next(); String topic = topicEntry.getKey(); - List queueDatas = topicEntry.getValue(); + Map queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSysFlag())) { + && TopicSysFlag.hasUnitFlag(queueDatas.values().iterator().next().getTopicSysFlag())) { topicList.getTopicList().add(topic); } } @@ -735,14 +1097,14 @@ public class RouteInfoManager { try { try { this.lock.readLock().lockInterruptibly(); - Iterator>> topicTableIt = + Iterator>> topicTableIt = this.topicQueueTable.entrySet().iterator(); while (topicTableIt.hasNext()) { - Entry> topicEntry = topicTableIt.next(); + Entry> topicEntry = topicTableIt.next(); String topic = topicEntry.getKey(); - List queueDatas = topicEntry.getValue(); + Map queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSysFlag())) { + && TopicSysFlag.hasUnitSubFlag(queueDatas.values().iterator().next().getTopicSysFlag())) { topicList.getTopicList().add(topic); } } @@ -761,15 +1123,15 @@ public class RouteInfoManager { try { try { this.lock.readLock().lockInterruptibly(); - Iterator>> topicTableIt = + Iterator>> topicTableIt = this.topicQueueTable.entrySet().iterator(); while (topicTableIt.hasNext()) { - Entry> topicEntry = topicTableIt.next(); + Entry> topicEntry = topicTableIt.next(); String topic = topicEntry.getKey(); - List queueDatas = topicEntry.getValue(); + Map queueDatas = topicEntry.getValue(); if (queueDatas != null && queueDatas.size() > 0 - && !TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSysFlag()) - && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSysFlag())) { + && !TopicSysFlag.hasUnitFlag(queueDatas.values().iterator().next().getTopicSysFlag()) + && TopicSysFlag.hasUnitSubFlag(queueDatas.values().iterator().next().getTopicSysFlag())) { topicList.getTopicList().add(topic); } } @@ -784,15 +1146,82 @@ public class RouteInfoManager { } } +/** + * broker address information + */ +class BrokerAddrInfo { + private String clusterName; + private String brokerAddr; + + private int hash; + + public BrokerAddrInfo(String clusterName, String brokerAddr) { + this.clusterName = clusterName; + this.brokerAddr = brokerAddr; + } + + public String getClusterName() { + return clusterName; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + public boolean isEmpty() { + return clusterName.isEmpty() && brokerAddr.isEmpty(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + + if (obj instanceof BrokerAddrInfo) { + BrokerAddrInfo addr = (BrokerAddrInfo) obj; + return clusterName.equals(addr.clusterName) && brokerAddr.equals(addr.brokerAddr); + } + return false; + } + + @Override + public int hashCode() { + int h = hash; + if (h == 0 && clusterName.length() + brokerAddr.length() > 0) { + for (int i = 0; i < clusterName.length(); i++) { + h = 31 * h + clusterName.charAt(i); + } + h = 31 * h + '_'; + for (int i = 0; i < brokerAddr.length(); i++) { + h = 31 * h + brokerAddr.charAt(i); + } + hash = h; + } + return h; + } + + @Override + public String toString() { + return "BrokerAddrInfo [clusterName=" + clusterName + ", brokerAddr=" + brokerAddr + "]"; + } +} + class BrokerLiveInfo { private long lastUpdateTimestamp; + private long heartbeatTimeoutMillis; private DataVersion dataVersion; private Channel channel; private String haServerAddr; - public BrokerLiveInfo(long lastUpdateTimestamp, DataVersion dataVersion, Channel channel, + public BrokerLiveInfo(long lastUpdateTimestamp, long heartbeatTimeoutMillis, DataVersion dataVersion, + Channel channel, String haServerAddr) { this.lastUpdateTimestamp = lastUpdateTimestamp; + this.heartbeatTimeoutMillis = heartbeatTimeoutMillis; this.dataVersion = dataVersion; this.channel = channel; this.haServerAddr = haServerAddr; @@ -806,6 +1235,14 @@ class BrokerLiveInfo { this.lastUpdateTimestamp = lastUpdateTimestamp; } + public long getHeartbeatTimeoutMillis() { + return heartbeatTimeoutMillis; + } + + public void setHeartbeatTimeoutMillis(long heartbeatTimeoutMillis) { + this.heartbeatTimeoutMillis = heartbeatTimeoutMillis; + } + public DataVersion getDataVersion() { return dataVersion; } @@ -836,3 +1273,39 @@ class BrokerLiveInfo { + ", channel=" + channel + ", haServerAddr=" + haServerAddr + "]"; } } + +class BrokerStatusChangeInfo { + Map brokerAddrs; + String offlineBrokerAddr; + String haBrokerAddr; + + public BrokerStatusChangeInfo(Map brokerAddrs, String offlineBrokerAddr, String haBrokerAddr) { + this.brokerAddrs = brokerAddrs; + this.offlineBrokerAddr = offlineBrokerAddr; + this.haBrokerAddr = haBrokerAddr; + } + + public Map getBrokerAddrs() { + return brokerAddrs; + } + + public void setBrokerAddrs(Map brokerAddrs) { + this.brokerAddrs = brokerAddrs; + } + + public String getOfflineBrokerAddr() { + return offlineBrokerAddr; + } + + public void setOfflineBrokerAddr(String offlineBrokerAddr) { + this.offlineBrokerAddr = offlineBrokerAddr; + } + + public String getHaBrokerAddr() { + return haBrokerAddr; + } + + public void setHaBrokerAddr(String haBrokerAddr) { + this.haBrokerAddr = haBrokerAddr; + } +} diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java new file mode 100644 index 0000000000..8f91d5b247 --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/RequestProcessorTest.java @@ -0,0 +1,563 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.namesrv.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.namesrv.NamesrvController; +import org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.assertj.core.util.Maps; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RequestProcessorTest { + private DefaultRequestProcessor defaultRequestProcessor; + + private ClientRequestProcessor clientRequestProcessor; + + private NamesrvController namesrvController; + + private NamesrvConfig namesrvConfig; + + private NettyServerConfig nettyServerConfig; + + private RouteInfoManager routeInfoManager; + + private InternalLogger logger; + + @Before + public void init() throws Exception { + namesrvConfig = new NamesrvConfig(); + namesrvConfig.setEnableAllTopicList(true); + nettyServerConfig = new NettyServerConfig(); + routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null); + + namesrvController = new NamesrvController(namesrvConfig, nettyServerConfig); + + Field field = NamesrvController.class.getDeclaredField("routeInfoManager"); + field.setAccessible(true); + field.set(namesrvController, routeInfoManager); + defaultRequestProcessor = new DefaultRequestProcessor(namesrvController); + + clientRequestProcessor = new ClientRequestProcessor(namesrvController); + + + registerRouteInfoManager(); + + logger = mock(InternalLogger.class); + setFinalStatic(DefaultRequestProcessor.class.getDeclaredField("log"), logger); + } + + @Test + public void testProcessRequest_PutKVConfig() throws RemotingCommandException { + PutKVConfigRequestHeader header = new PutKVConfigRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUT_KV_CONFIG, + header); + request.addExtField("namespace", "namespace"); + request.addExtField("key", "key"); + request.addExtField("value", "value"); + + RemotingCommand response = defaultRequestProcessor.processRequest(null, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + assertThat(namesrvController.getKvConfigManager().getKVConfig("namespace", "key")) + .isEqualTo("value"); + } + + @Test + public void testProcessRequest_GetKVConfigReturnNotNull() throws RemotingCommandException { + namesrvController.getKvConfigManager().putKVConfig("namespace", "key", "value"); + + GetKVConfigRequestHeader header = new GetKVConfigRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG, + header); + request.addExtField("namespace", "namespace"); + request.addExtField("key", "key"); + + RemotingCommand response = defaultRequestProcessor.processRequest(null, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response + .readCustomHeader(); + + assertThat(responseHeader.getValue()).isEqualTo("value"); + } + + @Test + public void testProcessRequest_GetKVConfigReturnNull() throws RemotingCommandException { + GetKVConfigRequestHeader header = new GetKVConfigRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG, + header); + request.addExtField("namespace", "namespace"); + request.addExtField("key", "key"); + + RemotingCommand response = defaultRequestProcessor.processRequest(null, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND); + assertThat(response.getRemark()).isEqualTo("No config item, Namespace: namespace Key: key"); + + GetKVConfigResponseHeader responseHeader = (GetKVConfigResponseHeader) response + .readCustomHeader(); + + assertThat(responseHeader.getValue()).isNull(); + } + + @Test + public void testProcessRequest_DeleteKVConfig() throws RemotingCommandException { + namesrvController.getKvConfigManager().putKVConfig("namespace", "key", "value"); + + DeleteKVConfigRequestHeader header = new DeleteKVConfigRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG, + header); + request.addExtField("namespace", "namespace"); + request.addExtField("key", "key"); + + RemotingCommand response = defaultRequestProcessor.processRequest(null, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + assertThat(namesrvController.getKvConfigManager().getKVConfig("namespace", "key")) + .isNull(); + } + + @Test + public void testProcessRequest_UnSupportedRequest() throws RemotingCommandException { + final RemotingCommand unSupportedRequest = RemotingCommand.createRequestCommand(99999, null); + final RemotingCommand response = defaultRequestProcessor.processRequest(null, unSupportedRequest); + + assertThat(response).isNotNull(); + assertThat(response.getCode()).isEqualTo(ResponseCode.REQUEST_CODE_NOT_SUPPORTED); + } + + @Test + public void testProcessRequest_RegisterBroker() throws RemotingCommandException, + NoSuchFieldException, IllegalAccessException { + RemotingCommand request = genSampleRegisterCmd(true); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + RouteInfoManager routes = namesrvController.getRouteInfoManager(); + Field brokerAddrTable = RouteInfoManager.class.getDeclaredField("brokerAddrTable"); + brokerAddrTable.setAccessible(true); + + BrokerData broker = new BrokerData(); + broker.setBrokerName("broker"); + broker.setBrokerAddrs((HashMap) Maps.newHashMap(new Long(2333), "10.10.1.1")); + + assertThat((Map) brokerAddrTable.get(routes)) + .contains(new HashMap.SimpleEntry("broker", broker)); + } + + /*@Test + public void testProcessRequest_RegisterBrokerLogicalQueue() throws Exception { + String cluster = "cluster"; + String broker1Name = "broker1"; + String broker1Addr = "10.10.1.1"; + String broker2Name = "broker2"; + String broker2Addr = "10.10.1.2"; + String topic = "foobar"; + + LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.ReadOnly, 0, 10, 100, 100, broker1Addr); + { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName(broker1Name); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.REGISTER_BROKER, header); + request.addExtField("brokerName", broker1Name); + request.addExtField("brokerAddr", broker1Addr); + request.addExtField("clusterName", cluster); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", String.valueOf(MixAll.MASTER_ID)); + request.setVersion(MQVersion.CURRENT_VERSION); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic)))); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(Collections.singletonMap(0, Lists.newArrayList( + queueRouteData1 + ))))); + topicConfigSerializeWrapper.setDataVersion(new DataVersion()); + RegisterBrokerBody requestBody = new RegisterBrokerBody(); + requestBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper); + requestBody.setFilterServerList(Lists.newArrayList()); + request.setBody(requestBody.encode()); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + } + LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); + LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(1, 100, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); + { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName(broker2Name); + RemotingCommand request = RemotingCommand.createRequestCommand( + RequestCode.REGISTER_BROKER, header); + request.addExtField("brokerName", broker2Name); + request.addExtField("brokerAddr", broker2Addr); + request.addExtField("clusterName", cluster); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", String.valueOf(MixAll.MASTER_ID)); + request.setVersion(MQVersion.CURRENT_VERSION); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(new ConcurrentHashMap<>(Collections.singletonMap(topic, new TopicConfig(topic)))); + topicConfigSerializeWrapper.setLogicalQueuesInfoMap(Maps.newHashMap(topic, new LogicalQueuesInfo(ImmutableMap.of( + 0, Collections.singletonList(queueRouteData2), + 1, Collections.singletonList(queueRouteData3) + )))); + topicConfigSerializeWrapper.setDataVersion(new DataVersion()); + RegisterBrokerBody requestBody = new RegisterBrokerBody(); + requestBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper); + requestBody.setFilterServerList(Lists.newArrayList()); + request.setBody(requestBody.encode()); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + } + + { + GetRouteInfoRequestHeader header = new GetRouteInfoRequestHeader(); + header.setTopic(topic); + header.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, header); + request.makeCustomHeaderToNet(); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + + TopicRouteDataNameSrv topicRouteDataNameSrv = JSON.parseObject(response.getBody(), TopicRouteDataNameSrv.class); + assertThat(topicRouteDataNameSrv).isNotNull(); + LogicalQueuesInfoUnordered logicalQueuesInfoUnordered = new LogicalQueuesInfoUnordered(); + logicalQueuesInfoUnordered.put(0, ImmutableMap.of( + new LogicalQueuesInfoUnordered.Key(queueRouteData1.getBrokerName(), queueRouteData1.getQueueId(), queueRouteData1.getOffsetDelta()), queueRouteData1, + new LogicalQueuesInfoUnordered.Key(queueRouteData2.getBrokerName(), queueRouteData2.getQueueId(), queueRouteData2.getOffsetDelta()), queueRouteData2 + )); + logicalQueuesInfoUnordered.put(1, ImmutableMap.of(new LogicalQueuesInfoUnordered.Key(queueRouteData3.getBrokerName(), queueRouteData3.getQueueId(), queueRouteData3.getOffsetDelta()), queueRouteData3)); + assertThat(topicRouteDataNameSrv.getLogicalQueuesInfoUnordered()).isEqualTo(logicalQueuesInfoUnordered); + } + } +*/ + @Test + public void testProcessRequest_RegisterBrokerWithFilterServer() throws RemotingCommandException, + NoSuchFieldException, IllegalAccessException { + RemotingCommand request = genSampleRegisterCmd(true); + + // version >= MQVersion.Version.V3_0_11.ordinal() to register with filter server + request.setVersion(100); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + RouteInfoManager routes = namesrvController.getRouteInfoManager(); + Field brokerAddrTable = RouteInfoManager.class.getDeclaredField("brokerAddrTable"); + brokerAddrTable.setAccessible(true); + + BrokerData broker = new BrokerData(); + broker.setBrokerName("broker"); + broker.setBrokerAddrs((HashMap) Maps.newHashMap(new Long(2333), "10.10.1.1")); + + assertThat((Map) brokerAddrTable.get(routes)) + .contains(new HashMap.SimpleEntry("broker", broker)); + } + + @Test + public void testProcessRequest_UnregisterBroker() throws RemotingCommandException, NoSuchFieldException, IllegalAccessException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + + //Register broker + RemotingCommand regRequest = genSampleRegisterCmd(true); + defaultRequestProcessor.processRequest(ctx, regRequest); + + //Unregister broker + RemotingCommand unregRequest = genSampleRegisterCmd(false); + RemotingCommand unregResponse = defaultRequestProcessor.processRequest(ctx, unregRequest); + + assertThat(unregResponse.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(unregResponse.getRemark()).isNull(); + + RouteInfoManager routes = namesrvController.getRouteInfoManager(); + Field brokerAddrTable = RouteInfoManager.class.getDeclaredField("brokerAddrTable"); + brokerAddrTable.setAccessible(true); + + assertThat((Map) brokerAddrTable.get(routes)).isNotEmpty(); + } + + @Test + public void testGetAllTopicList() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(channel.remoteAddress()).thenReturn(null); + when(ctx.channel()).thenReturn(channel); + + namesrvController.getNamesrvConfig().setEnableAllTopicList(true); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER, null); + + RemotingCommand response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(response.getRemark()).isNull(); + + namesrvController.getNamesrvConfig().setEnableAllTopicList(false); + + response = defaultRequestProcessor.processRequest(ctx, request); + + assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + } + + @Test + public void testGetRouteInfoByTopic() throws Exception { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC); + RemotingCommand remotingCommandSuccess = clientRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommandSuccess.getCode()).isEqualTo(ResponseCode.SUCCESS); + request.getExtFields().put("topic", "test"); + RemotingCommand remotingCommandNoTopicRouteInfo = clientRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommandNoTopicRouteInfo.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST); + } + + @Test + public void testGetBrokerClusterInfo() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_BROKER_CLUSTER_INFO); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testWipeWritePermOfBroker() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.WIPE_WRITE_PERM_OF_BROKER); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetAllTopicListFromNameserver() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(mock(Channel.class)); + when(ctx.channel().remoteAddress()).thenReturn(new InetSocketAddress(123)); + RemotingCommand request = getRemotingCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testDeleteTopicInNamesrv() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetKVListByNamespace() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_KVLIST_BY_NAMESPACE); + request.addExtField("namespace", "default-namespace-1"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.QUERY_NOT_FOUND); + namesrvController.getKvConfigManager().putKVConfig("default-namespace-1", "key", "value"); + RemotingCommand remotingCommandSuccess = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommandSuccess.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetTopicsByCluster() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_TOPICS_BY_CLUSTER); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetSystemTopicListFromNs() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetUnitTopicList() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_UNIT_TOPIC_LIST); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetHasUnitSubTopicList() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetHasUnitSubUnUnitTopicList() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testUpdateConfig() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.UPDATE_NAMESRV_CONFIG); + request.addExtField("cluster", "default-cluster"); + Map propertiesMap = new HashMap<>(); + propertiesMap.put("key", "value"); + request.setBody(propertiesMap.toString().getBytes()); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testGetConfig() throws RemotingCommandException { + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + when(ctx.channel()).thenReturn(null); + RemotingCommand request = getRemotingCommand(RequestCode.GET_NAMESRV_CONFIG); + request.addExtField("cluster", "default-cluster"); + RemotingCommand remotingCommand = defaultRequestProcessor.processRequest(ctx, request); + assertThat(remotingCommand.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + private RemotingCommand getRemotingCommand(int code) { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName("broker"); + RemotingCommand request = RemotingCommand.createRequestCommand(code, header); + request.addExtField("brokerName", "broker"); + request.addExtField("brokerAddr", "10.10.1.1"); + request.addExtField("clusterName", "cluster"); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", "2333"); + request.addExtField("topic", "unit-test"); + return request; + } + + private static RemotingCommand genSampleRegisterCmd(boolean reg) { + RegisterBrokerRequestHeader header = new RegisterBrokerRequestHeader(); + header.setBrokerName("broker"); + RemotingCommand request = RemotingCommand.createRequestCommand( + reg ? RequestCode.REGISTER_BROKER : RequestCode.UNREGISTER_BROKER, header); + request.addExtField("brokerName", "broker"); + request.addExtField("brokerAddr", "10.10.1.1"); + request.addExtField("clusterName", "cluster"); + request.addExtField("haServerAddr", "10.10.2.1"); + request.addExtField("brokerId", "2333"); + return request; + } + + private static void setFinalStatic(Field field, Object newValue) throws Exception { + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(null, newValue); + } + + private void registerRouteInfoManager() { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName("unit-test"); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put("unit-test", topicConfig); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); + Channel channel = mock(Channel.class); + RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker("default-cluster", "127.0.0.1:10911", "default-broker", 1234, "127.0.0.1:1001", + null, topicConfigSerializeWrapper, new ArrayList(), channel); + + } +} \ No newline at end of file diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/GetRouteInfoBenchmark.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/GetRouteInfoBenchmark.java new file mode 100644 index 0000000000..0ea8b054a9 --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/GetRouteInfoBenchmark.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.routeinfo; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import io.netty.channel.Channel; +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.utils.ThreadUtils; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.slf4j.LoggerFactory; + +import static org.mockito.Mockito.mock; + +@BenchmarkMode(Mode.SampleTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class GetRouteInfoBenchmark { + private RouteInfoManager routeInfoManager; + private String[] topicList = new String[40000]; + private ExecutorService es = Executors.newCachedThreadPool(); + + @Setup + public void setup() throws InterruptedException, JoranException { + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + //https://logback.qos.ch/manual/configuration.html + lc.setPackagingDataEnabled(false); + + routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null); + + // Init 4 clusters and 8 brokers in each cluster + // Each cluster has 10000 topics + + for (int i = 0; i < 40000; i++) { + final String topic = RandomStringUtils.randomAlphabetic(32) + i; + topicList[i] = topic; + } + + for (int i = 0; i < 4; i++) { + // Cluster iteration + final String clusterName = "Default-Cluster-" + i; + for (int j = 0; j < 8; j++) { + // broker iteration + final int startTopicIndex = i * 10000; + final String brokerName = "Default-Broker-" + j; + final String brokerAddr = "127.0.0.1:500" + i * j; + es.submit(new Runnable() { + @Override + public void run() { + DataVersion dataVersion = new DataVersion(); + dataVersion.setCounter(new AtomicLong(10L)); + dataVersion.setTimestamp(100L); + + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + + for (int k = startTopicIndex; k < startTopicIndex + 10000; k++) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topicList[k]); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put(topicList[k], topicConfig); + } + + while (true) { + try { + TimeUnit.MILLISECONDS.sleep(new Random().nextInt(100)); + } catch (InterruptedException ignored) { + } + + dataVersion.nextVersion(); + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(dataVersion); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); + Channel channel = mock(Channel.class); + + routeInfoManager.registerBroker(clusterName, brokerAddr, brokerName, 0, brokerAddr, + null, topicConfigSerializeWrapper, new ArrayList(), channel); + } + } + }); + } + } + + // Wait threads startup + TimeUnit.SECONDS.sleep(3); + } + + @TearDown + public void tearDown() { + ThreadUtils.shutdownGracefully(es, 3, TimeUnit.SECONDS); + } + + @Benchmark + @Fork(value = 2) + @Measurement(iterations = 10, time = 10) + @Warmup(iterations = 10, time = 1) + @Threads(4) // Assume we have 128 clients try to pick up route data concurrently + public void pickupTopicRouteData() { + routeInfoManager.pickupTopicRouteData(topicList[new Random().nextInt(40000)]); + } + + public static void main(String[] args) throws Exception { + org.openjdk.jmh.Main.main(args); + } +} diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RegisterBrokerBenchmark.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RegisterBrokerBenchmark.java new file mode 100644 index 0000000000..f64211582a --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RegisterBrokerBenchmark.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.routeinfo; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import io.netty.channel.Channel; +import java.util.ArrayList; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.utils.ThreadUtils; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; +import org.slf4j.LoggerFactory; + +import static org.mockito.Mockito.mock; + +@BenchmarkMode(Mode.SampleTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class RegisterBrokerBenchmark { + private RouteInfoManager routeInfoManager; + private String[] topicList = new String[40000]; + private ConcurrentHashMap[] topicConfigMaps = new ConcurrentHashMap[32]; + private DataVersion[] dataVersions = new DataVersion[32]; + private ExecutorService es = Executors.newCachedThreadPool(); + private AtomicLong brokerIndex = new AtomicLong(0); + + @Setup + public void setup() throws InterruptedException, JoranException { + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + //https://logback.qos.ch/manual/configuration.html + lc.setPackagingDataEnabled(false); + + routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null); + + // Init 4 clusters and 8 brokers in each cluster + // Each cluster has 10000 topics + + for (int i = 0; i < 40000; i++) { + final String topic = RandomStringUtils.randomAlphabetic(32) + i; + topicList[i] = topic; + } + + for (int i = 0; i < 4; i++) { + // Cluster iteration + final String clusterName = "Default-Cluster-" + i; + for (int j = 0; j < 8; j++) { + // broker iteration + final int startTopicIndex = i * 10000; + final String brokerName = "Default-Broker-" + j; + final String brokerAddr = "127.0.0.1:500" + (i * 8 + j); + + topicConfigMaps[i * 8 + j] = new ConcurrentHashMap<>(); + for (int k = startTopicIndex; k < startTopicIndex + 10000; k++) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topicList[k]); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigMaps[i * 8 + j].put(topicList[k], topicConfig); + } + + DataVersion dataVersion = new DataVersion(); + dataVersion.setCounter(new AtomicLong(10L)); + dataVersion.setTimestamp(100L); + + dataVersions[i * 8 + j] = dataVersion; + } + } + + // Init 32 threads to pick up route info + for (int i = 0; i < 32; i++) { + es.submit(new Runnable() { + @Override + public void run() { + routeInfoManager.pickupTopicRouteData(topicList[new Random().nextInt(40000)]); + try { + TimeUnit.MILLISECONDS.sleep(new Random().nextInt(10)); + } catch (InterruptedException ignored) { + } + } + }); + } + } + + @TearDown + public void tearDown() { + ThreadUtils.shutdownGracefully(es, 3, TimeUnit.SECONDS); + } + + @Benchmark + @Fork(value = 2) + @Measurement(iterations = 10, time = 10) + @Warmup(iterations = 10, time = 1) + @Threads(32) // Assume we have 128 clients try to pick up route data concurrently + public void registerBroker() { + final long index = Math.abs(brokerIndex.getAndIncrement() % 32); + dataVersions[(int) index].nextVersion(); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(dataVersions[(int) index]); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigMaps[(int) index]); + Channel channel = mock(Channel.class); + + routeInfoManager.registerBroker("DefaultCluster" + index, + "127.0.0.1:500" + index, + "DefaultBroker" + index, 0, "127.0.0.1:400" + index, + null, + topicConfigSerializeWrapper, new ArrayList(), channel); + } + + @Benchmark + @Fork(value = 2) + @Measurement(iterations = 10, time = 10) + @Warmup(iterations = 10, time = 1) + @Threads(32) // Assume we have 128 clients try to pick up route data concurrently + @BenchmarkMode(Mode.Throughput) + public void registerBroker_Throughput() { + final long index = Math.abs(brokerIndex.getAndIncrement() % 32); + dataVersions[(int) index].nextVersion(); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(dataVersions[(int) index]); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigMaps[(int) index]); + Channel channel = mock(Channel.class); + + routeInfoManager.registerBroker("DefaultCluster" + index, + "127.0.0.1:500" + index, + "DefaultBroker" + index, 0, "127.0.0.1:400" + index, + null, + topicConfigSerializeWrapper, new ArrayList(), channel); + } + + public static void main(String[] args) throws Exception { + org.openjdk.jmh.Main.main(args); + } +} diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java index e0d9e1871c..4acfd32150 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManagerTest.java @@ -17,8 +17,12 @@ package org.apache.rocketmq.namesrv.routeinfo; import io.netty.channel.Channel; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.route.QueueData; @@ -31,7 +35,6 @@ import org.junit.Test; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; import static org.assertj.core.api.Assertions.assertThat; @@ -43,12 +46,14 @@ public class RouteInfoManagerTest { @Before public void setup() { - routeInfoManager = new RouteInfoManager(); + routeInfoManager = new RouteInfoManager(new NamesrvConfig(), null); + routeInfoManager.start(); testRegisterBroker(); } @After public void terminate() { + routeInfoManager.shutdown(); routeInfoManager.printAllPeriodically(); routeInfoManager.unregisterBroker("default-cluster", "127.0.0.1:10911", "default-broker", 1234); } @@ -59,6 +64,55 @@ public class RouteInfoManagerTest { assertThat(clusterInfo).isNotNull(); } + @Test + public void testQueryBrokerTopicConfig() { + { + DataVersion targetVersion = new DataVersion(); + targetVersion.setCounter(new AtomicLong(10L)); + targetVersion.setTimestamp(100L); + + DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig("default-cluster", "127.0.0.1:10911"); + assertThat(dataVersion.equals(targetVersion)).isTrue(); + } + + { + // register broker default-cluster-1 with the same addr, then test + DataVersion targetVersion = new DataVersion(); + targetVersion.setCounter(new AtomicLong(20L)); + targetVersion.setTimestamp(200L); + + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName("unit-test"); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put("unit-test-1", topicConfig); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(targetVersion); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); + Channel channel = mock(Channel.class); + RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker("default-cluster-1", "127.0.0.1:10911", "default-broker-1", 1234, "127.0.0.1:1001", + null, topicConfigSerializeWrapper, new ArrayList(), channel); + assertThat(registerBrokerResult).isNotNull(); + + DataVersion dataVersion0 = routeInfoManager.queryBrokerTopicConfig("default-cluster", "127.0.0.1:10911"); + assertThat(targetVersion.equals(dataVersion0)).isFalse(); + + DataVersion dataVersion1 = routeInfoManager.queryBrokerTopicConfig("default-cluster-1", "127.0.0.1:10911"); + assertThat(targetVersion.equals(dataVersion1)).isTrue(); + } + + // unregister broker default-cluster-1, then test + { + routeInfoManager.unregisterBroker("default-cluster-1", "127.0.0.1:10911", "default-broker-1", 1234); + assertThat(null != routeInfoManager.queryBrokerTopicConfig("default-cluster", "127.0.0.1:10911")).isTrue(); + assertThat(null == routeInfoManager.queryBrokerTopicConfig("default-cluster-1", "127.0.0.1:10911")).isTrue(); + } + } + @Test public void testGetAllTopicList() { byte[] topicInfo = routeInfoManager.getAllTopicList(); @@ -68,7 +122,10 @@ public class RouteInfoManagerTest { @Test public void testRegisterBroker() { - TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + DataVersion dataVersion = new DataVersion(); + dataVersion.setCounter(new AtomicLong(10L)); + dataVersion.setTimestamp(100L); + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); TopicConfig topicConfig = new TopicConfig(); topicConfig.setWriteQueueNums(8); @@ -77,22 +134,26 @@ public class RouteInfoManagerTest { topicConfig.setReadQueueNums(8); topicConfig.setOrder(false); topicConfigConcurrentHashMap.put("unit-test", topicConfig); + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(dataVersion); topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); Channel channel = mock(Channel.class); RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker("default-cluster", "127.0.0.1:10911", "default-broker", 1234, "127.0.0.1:1001", - topicConfigSerializeWrapper, new ArrayList(), channel); + null, topicConfigSerializeWrapper, new ArrayList(), channel); assertThat(registerBrokerResult).isNotNull(); } @Test public void testWipeWritePermOfBrokerByLock() throws Exception { - List qdList = new ArrayList<>(); + Map qdMap = new HashMap<>(); + QueueData qd = new QueueData(); qd.setPerm(PermName.PERM_READ | PermName.PERM_WRITE); qd.setBrokerName("broker-a"); - qdList.add(qd); - HashMap> topicQueueTable = new HashMap<>(); - topicQueueTable.put("topic-a", qdList); + qdMap.put("broker-a",qd); + HashMap> topicQueueTable = new HashMap<>(); + topicQueueTable.put("topic-a", qdMap); Field filed = RouteInfoManager.class.getDeclaredField("topicQueueTable"); filed.setAccessible(true); @@ -142,13 +203,13 @@ public class RouteInfoManagerTest { @Test public void testAddWritePermOfBrokerByLock() throws Exception { - List qdList = new ArrayList<>(); + Map qdMap = new HashMap<>(); QueueData qd = new QueueData(); qd.setPerm(PermName.PERM_READ); qd.setBrokerName("broker-a"); - qdList.add(qd); - HashMap> topicQueueTable = new HashMap<>(); - topicQueueTable.put("topic-a", qdList); + qdMap.put("broker-a",qd); + HashMap> topicQueueTable = new HashMap<>(); + topicQueueTable.put("topic-a", qdMap); Field filed = RouteInfoManager.class.getDeclaredField("topicQueueTable"); filed.setAccessible(true); diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager_NewTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager_NewTest.java new file mode 100644 index 0000000000..712f9d93a6 --- /dev/null +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager_NewTest.java @@ -0,0 +1,783 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.namesrv.routeinfo; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.Sets; +import io.netty.channel.Channel; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.namesrv.NamesrvConfig; +import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.body.TopicList; +import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Spy; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +public class RouteInfoManager_NewTest { + private RouteInfoManager routeInfoManager; + private static final String DEFAULT_CLUSTER = "Default_Cluster"; + private static final String DEFAULT_BROKER = "Default_Broker"; + private static final String DEFAULT_ADDR_PREFIX = "127.0.0.1:"; + private static final String DEFAULT_ADDR = DEFAULT_ADDR_PREFIX + "10911"; + + // Synced from RouteInfoManager + private static final int BROKER_CHANNEL_EXPIRED_TIME = 1000 * 60 * 2; + + @Spy + private static NamesrvConfig config = spy(new NamesrvConfig()); + + @Before + public void setup() { + config.setSupportActingMaster(true); + routeInfoManager = new RouteInfoManager(config, null); + routeInfoManager.start(); + } + + @After + public void tearDown() throws Exception { + routeInfoManager.shutdown(); + } + + @Test + public void getAllClusterInfo() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker() + .cluster("AnotherCluster") + .name("AnotherBroker") + .addr(DEFAULT_ADDR_PREFIX + 30911), "TestTopic1"); + + final byte[] content = routeInfoManager.getAllClusterInfo(); + + final ClusterInfo clusterInfo = ClusterInfo.decode(content, ClusterInfo.class); + + assertThat(clusterInfo.retrieveAllClusterNames()).contains(DEFAULT_CLUSTER, "AnotherCluster"); + assertThat(clusterInfo.getBrokerAddrTable().keySet()).contains(DEFAULT_BROKER, "AnotherBroker"); + + final List addrList = Arrays.asList(clusterInfo.getBrokerAddrTable().get(DEFAULT_BROKER).getBrokerAddrs().get(0L), + clusterInfo.getBrokerAddrTable().get("AnotherBroker").getBrokerAddrs().get(0L)); + assertThat(addrList).contains(DEFAULT_ADDR, DEFAULT_ADDR_PREFIX + 30911); + } + + @Test + public void deleteTopic() { + String testTopic = "TestTopic"; + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + + routeInfoManager.deleteTopic(testTopic); + + assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNull(); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker().cluster("AnotherCluster").name("AnotherBroker"), + testTopic); + + assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().size()).isEqualTo(2); + routeInfoManager.deleteTopic(testTopic, DEFAULT_CLUSTER); + + assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData(testTopic).getBrokerDatas().get(0).getBrokerName()).isEqualTo("AnotherBroker"); + } + + @Test + public void getAllTopicList() { + byte[] content = routeInfoManager.getAllTopicList(); + + TopicList topicList = TopicList.decode(content, TopicList.class); + assertThat(topicList.getTopicList()).isEmpty(); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic", "TestTopic1", "TestTopic2"); + + content = routeInfoManager.getAllTopicList(); + + topicList = TopicList.decode(content, TopicList.class); + assertThat(topicList.getTopicList()).contains("TestTopic", "TestTopic1", "TestTopic2"); + } + + @Test + public void registerBroker() { + // Register master broker + final RegisterBrokerResult masterResult = registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + + assertThat(masterResult).isNotNull(); + assertThat(masterResult.getHaServerAddr()).isNull(); + assertThat(masterResult.getMasterAddr()).isNull(); + + // Register slave broker + + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.defaultBroker() + .id(1).addr(DEFAULT_ADDR_PREFIX + 30911).haAddr(DEFAULT_ADDR_PREFIX + 40911); + + final RegisterBrokerResult slaveResult = registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + assertThat(slaveResult).isNotNull(); + assertThat(slaveResult.getHaServerAddr()).isEqualTo(DEFAULT_ADDR_PREFIX + 20911); + assertThat(slaveResult.getMasterAddr()).isEqualTo(DEFAULT_ADDR); + } + + @Test + public void unregisterBroker() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic", "TestTopic1", "TestTopic2"); + routeInfoManager.unregisterBroker(DEFAULT_CLUSTER, DEFAULT_ADDR, DEFAULT_BROKER, 0); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic2")).isNull(); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic", "TestTopic1", "TestTopic2"); + + routeInfoManager.submitUnRegisterBrokerRequest(BrokerBasicInfo.defaultBroker().unRegisterRequest()); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic2")).isNull(); + } + + @Test + public void registerSlaveBroker() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L); + + registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr); + } + + @Test + public void createNewTopic() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), "TestTopic"); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic", "TestTopic1"); + + System.out.println(JSON.toJSONString(routeInfoManager.pickupTopicRouteData("TestTopic1"))); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr); + } + + @Test + public void switchBrokerRole() { + final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false); + registerBrokerWithNormalTopic(masterBroker, "TestTopic"); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + // Master Down + routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(slaveBroker.brokerAddr); + + // Switch slave to master + slaveBroker.id(0).dataVersion.nextVersion(); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(0L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(slaveBroker.brokerAddr); + + // Old master switch to slave + masterBroker.id(1).dataVersion.nextVersion(); + registerBrokerWithNormalTopic(masterBroker, "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr); + } + + @Test + public void unRegisterSlaveBroker() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker(); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + routeInfoManager.unregisterBroker(slaveBroker.clusterName, slaveBroker.brokerAddr, slaveBroker.brokerName, 1); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0) + .getBrokerAddrs().get(0L)).isEqualTo(BrokerBasicInfo.defaultBroker().brokerAddr); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + routeInfoManager.submitUnRegisterBrokerRequest(slaveBroker.unRegisterRequest()); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0) + .getBrokerAddrs().get(0L)).isEqualTo(BrokerBasicInfo.defaultBroker().brokerAddr); + } + + @Test + public void unRegisterMasterBroker() { + final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker(); + masterBroker.enableActingMaster = true; + registerBrokerWithNormalTopic(masterBroker, "TestTopic"); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker(); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0) + .getBrokerAddrs().get(0L)).isEqualTo(slaveBroker.brokerAddr); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().get(0).getPerm()).isEqualTo(PermName.PERM_READ); + } + + @Test + public void unRegisterMasterBrokerOldVersion() { + final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker(); + masterBroker.enableActingMaster = false; + registerBrokerWithNormalTopic(masterBroker, "TestTopic"); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker(); + slaveBroker.enableActingMaster = false; + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + routeInfoManager.unregisterBroker(masterBroker.clusterName, masterBroker.brokerAddr, masterBroker.brokerName, 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0) + .getBrokerAddrs().get(0L)).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().size()).isEqualTo(1); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().get(0).getPerm()).isEqualTo(PermName.PERM_READ | PermName.PERM_WRITE); + } + + @Test + public void submitMultiUnRegisterRequests() { + final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker(); + final BrokerBasicInfo master2 = BrokerBasicInfo.defaultBroker().name(DEFAULT_BROKER + 1).addr(DEFAULT_ADDR + 9); + registerBrokerWithNormalTopic(master1, "TestTopic1"); + registerBrokerWithNormalTopic(master2, "TestTopic2"); + + routeInfoManager.submitUnRegisterBrokerRequest(master1.unRegisterRequest()); + routeInfoManager.submitUnRegisterBrokerRequest(master2.unRegisterRequest()); + + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic2")).isNull(); + } + + @Test + public void isBrokerTopicConfigChanged() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + + final DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig(DEFAULT_CLUSTER, DEFAULT_ADDR); + + assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, dataVersion)).isFalse(); + + DataVersion newVersion = new DataVersion(); + newVersion.setTimestamp(System.currentTimeMillis() + 1000); + newVersion.setCounter(new AtomicLong(dataVersion.getCounter().get())); + + assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, newVersion)).isTrue(); + + newVersion = new DataVersion(); + newVersion.setTimestamp(dataVersion.getTimestamp()); + newVersion.setCounter(new AtomicLong(dataVersion.getCounter().get() + 1)); + + assertThat(routeInfoManager.isBrokerTopicConfigChanged(DEFAULT_CLUSTER, DEFAULT_ADDR, newVersion)).isTrue(); + } + + @Test + public void isTopicConfigChanged() { + final BrokerBasicInfo brokerInfo = BrokerBasicInfo.defaultBroker(); + assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER, + DEFAULT_ADDR, + brokerInfo.dataVersion, + DEFAULT_BROKER, "TestTopic")).isTrue(); + + registerBrokerWithNormalTopic(brokerInfo, "TestTopic"); + + assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER, + DEFAULT_ADDR, + brokerInfo.dataVersion, + DEFAULT_BROKER, "TestTopic")).isFalse(); + + assertThat(routeInfoManager.isTopicConfigChanged(DEFAULT_CLUSTER, + DEFAULT_ADDR, + brokerInfo.dataVersion, + DEFAULT_BROKER, "TestTopic1")).isTrue(); + } + + @Test + public void queryBrokerTopicConfig() { + final BrokerBasicInfo basicInfo = BrokerBasicInfo.defaultBroker(); + registerBrokerWithNormalTopic(basicInfo, "TestTopic"); + + final DataVersion dataVersion = routeInfoManager.queryBrokerTopicConfig(DEFAULT_CLUSTER, DEFAULT_ADDR); + + assertThat(basicInfo.dataVersion.equals(dataVersion)).isTrue(); + } + + @Test + public void wipeWritePermOfBrokerByLock() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().get(0).getPerm()).isEqualTo(6); + + routeInfoManager.wipeWritePermOfBrokerByLock(DEFAULT_BROKER); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getQueueDatas().get(0).getPerm()).isEqualTo(4); + } + + @Test + public void pickupTopicRouteData() { + String testTopic = "TestTopic"; + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + + TopicRouteData data = routeInfoManager.pickupTopicRouteData(testTopic); + assertThat(data.getBrokerDatas().size()).isEqualTo(1); + assertThat(data.getBrokerDatas().get(0).getBrokerName()).isEqualTo(DEFAULT_BROKER); + assertThat(data.getBrokerDatas().get(0).getBrokerAddrs().get(0L)).isEqualTo(DEFAULT_ADDR); + assertThat(data.getQueueDatas().size()).isEqualTo(1); + assertThat(data.getQueueDatas().get(0).getBrokerName()).isEqualTo(DEFAULT_BROKER); + assertThat(data.getQueueDatas().get(0).getReadQueueNums()).isEqualTo(8); + assertThat(data.getQueueDatas().get(0).getWriteQueueNums()).isEqualTo(8); + assertThat(data.getQueueDatas().get(0).getPerm()).isEqualTo(6); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker().name("AnotherBroker"), testTopic); + data = routeInfoManager.pickupTopicRouteData(testTopic); + + assertThat(data.getBrokerDatas().size()).isEqualTo(2); + assertThat(data.getQueueDatas().size()).isEqualTo(2); + + List brokerList = + Arrays.asList(data.getBrokerDatas().get(0).getBrokerName(), data.getBrokerDatas().get(1).getBrokerName()); + assertThat(brokerList).contains(DEFAULT_BROKER, "AnotherBroker"); + + brokerList = + Arrays.asList(data.getQueueDatas().get(0).getBrokerName(), data.getQueueDatas().get(1).getBrokerName()); + assertThat(brokerList).contains(DEFAULT_BROKER, "AnotherBroker"); + } + + @Test + public void pickupTopicRouteDataWithSlave() { + String testTopic = "TestTopic"; + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic); + + TopicRouteData routeData = routeInfoManager.pickupTopicRouteData(testTopic); + + assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(2); + assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isTrue(); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest())); + routeData = routeInfoManager.pickupTopicRouteData(testTopic); + + assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(1); + assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isFalse(); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + routeData = routeInfoManager.pickupTopicRouteData(testTopic); + + assertThat(routeData.getBrokerDatas().get(0).getBrokerAddrs()).hasSize(2); + assertThat(PermName.isWriteable(routeData.getQueueDatas().get(0).getPerm())).isTrue(); + } + + @Test + public void scanNotActiveBroker() { + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), "TestTopic"); + routeInfoManager.scanNotActiveBroker(); + } + + @Test + public void pickupPartitionOrderTopicRouteData() { + String orderTopic = "PartitionOrderTopicTest"; + + // Case 1: Register global order topic with slave + registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic); + + TopicRouteData orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic); + + // Acting master check + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsOnlyKeys(MixAll.MASTER_ID); + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr); + assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse(); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest())); + + // Case 2: Register global order topic with master and slave, then unregister master + + registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic); + registerBrokerWithOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic); + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest())); + + orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic); + + // Acting master check + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsOnlyKeys(MixAll.MASTER_ID); + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr); + assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse(); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest())); + + // Case 3: Register two broker groups, only one group enable acting master + registerBrokerWithOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic); + registerBrokerWithOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic); + + final BrokerBasicInfo master1 = BrokerBasicInfo.defaultBroker().name(DEFAULT_BROKER + "_ANOTHER"); + final BrokerBasicInfo slave1 = BrokerBasicInfo.slaveBroker().name(DEFAULT_BROKER + "_ANOTHER"); + + registerBrokerWithOrderTopic(master1, orderTopic); + registerBrokerWithOrderTopic(slave1, orderTopic); + + orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic); + + assertThat(orderRoute.getBrokerDatas()).hasSize(2); + assertThat(orderRoute.getQueueDatas()).hasSize(2); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest())); + orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic); + + assertThat(orderRoute.getBrokerDatas()).hasSize(2); + assertThat(orderRoute.getQueueDatas()).hasSize(2); + + for (final BrokerData brokerData : orderRoute.getBrokerDatas()) { + if (brokerData.getBrokerAddrs().size() == 1) { + assertThat(brokerData.getBrokerAddrs()).containsOnlyKeys(MixAll.MASTER_ID); + assertThat(brokerData.getBrokerAddrs()).containsValue(BrokerBasicInfo.slaveBroker().brokerAddr); + } else if (brokerData.getBrokerAddrs().size() == 2) { + assertThat(brokerData.getBrokerAddrs()).containsKeys(MixAll.MASTER_ID, (long) slave1.brokerId); + assertThat(brokerData.getBrokerAddrs()).containsValues(master1.brokerAddr, slave1.brokerAddr); + } else { + throw new RuntimeException("Shouldn't reach here"); + } + } + } + + @Test + public void pickupGlobalOrderTopicRouteData() { + String orderTopic = "GlobalOrderTopicTest"; + + // Case 1: Register global order topic with slave + registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic); + + TopicRouteData orderRoute = routeInfoManager.pickupTopicRouteData(orderTopic); + + // Acting master check + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsOnlyKeys(MixAll.MASTER_ID); + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr); + assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse(); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest())); + + // Case 2: Register global order topic with master and slave, then unregister master + + registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.slaveBroker(), orderTopic); + registerBrokerWithGlobalOrderTopic(BrokerBasicInfo.defaultBroker(), orderTopic); + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest())); + + // Acting master check + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsOnlyKeys(MixAll.MASTER_ID); + assertThat(orderRoute.getBrokerDatas().get(0).getBrokerAddrs()) + .containsValue(BrokerBasicInfo.slaveBroker().brokerAddr); + assertThat(PermName.isWriteable(orderRoute.getQueueDatas().get(0).getPerm())).isFalse(); + } + + @Test + public void registerOnlySlaveBroker() { + final String testTopic = "TestTopic"; + + // Case 1: Only slave broker + registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic); + assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull(); + int topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm(); + assertThat(PermName.isWriteable(topicPerm)).isFalse(); + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.slaveBroker().unRegisterRequest())); + + // Case 2: Register master, and slave, then unregister master, finally recover master + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + registerBrokerWithNormalTopic(BrokerBasicInfo.slaveBroker(), testTopic); + + assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull(); + topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm(); + assertThat(PermName.isWriteable(topicPerm)).isTrue(); + + routeInfoManager.unRegisterBroker(Sets.newHashSet(BrokerBasicInfo.defaultBroker().unRegisterRequest())); + assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull(); + topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm(); + assertThat(PermName.isWriteable(topicPerm)).isFalse(); + + registerBrokerWithNormalTopic(BrokerBasicInfo.defaultBroker(), testTopic); + assertThat(routeInfoManager.pickupTopicRouteData(testTopic)).isNotNull(); + topicPerm = routeInfoManager.pickupTopicRouteData(testTopic).getQueueDatas().get(0).getPerm(); + assertThat(PermName.isWriteable(topicPerm)).isTrue(); + } + + @Test + public void onChannelDestroy() { + Channel channel = mock(Channel.class); + + registerBroker(BrokerBasicInfo.defaultBroker(), channel, null, "TestTopic", "TestTopic1"); + routeInfoManager.onChannelDestroy(channel); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic1")).isNull(); + + final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false); + + Channel masterChannel = mock(Channel.class); + Channel slaveChannel = mock(Channel.class); + + registerBroker(masterBroker, masterChannel, null, "TestTopic"); + registerBroker(slaveBroker, slaveChannel, null, "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(masterBroker.brokerAddr, slaveBroker.brokerAddr); + + routeInfoManager.onChannelDestroy(masterChannel); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(slaveBroker.brokerAddr); + + routeInfoManager.onChannelDestroy(slaveChannel); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic")).isNull(); + } + + @Test + public void switchBrokerRole_ChannelDestroy() { + final BrokerBasicInfo masterBroker = BrokerBasicInfo.defaultBroker().enableActingMaster(false); + final BrokerBasicInfo slaveBroker = BrokerBasicInfo.slaveBroker().enableActingMaster(false); + + Channel masterChannel = mock(Channel.class); + Channel slaveChannel = mock(Channel.class); + + registerBroker(masterBroker, masterChannel, null, "TestTopic"); + registerBroker(slaveBroker, slaveChannel, null, "TestTopic"); + + // Master Down + routeInfoManager.onChannelDestroy(masterChannel); + await().atMost(Duration.ofSeconds(5)).until(() -> routeInfoManager.blockedUnRegisterRequests() == 0); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(slaveBroker.brokerAddr); + + // Switch slave to master + slaveBroker.id(0).dataVersion.nextVersion(); + registerBrokerWithNormalTopic(slaveBroker, "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsOnlyKeys(0L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(slaveBroker.brokerAddr); + + // Old master switch to slave + masterBroker.id(1).dataVersion.nextVersion(); + registerBrokerWithNormalTopic(masterBroker, "TestTopic"); + + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()).containsKeys(0L, 1L); + assertThat(routeInfoManager.pickupTopicRouteData("TestTopic").getBrokerDatas().get(0).getBrokerAddrs()) + .containsValues(BrokerBasicInfo.defaultBroker().brokerAddr, BrokerBasicInfo.slaveBroker().brokerAddr); + } + + private RegisterBrokerResult registerBrokerWithNormalTopic(BrokerBasicInfo brokerInfo, String... topics) { + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + + return registerBroker(brokerInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics); + } + + private RegisterBrokerResult registerBrokerWithOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) { + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(true); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + return registerBroker(brokerBasicInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics); + } + + private RegisterBrokerResult registerBrokerWithGlobalOrderTopic(BrokerBasicInfo brokerBasicInfo, String... topics) { + ConcurrentHashMap topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(1); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(1); + topicConfig.setOrder(true); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + return registerBroker(brokerBasicInfo, mock(Channel.class), topicConfigConcurrentHashMap, topics); + } + + private RegisterBrokerResult registerBroker(BrokerBasicInfo brokerInfo, Channel channel, + ConcurrentMap topicConfigConcurrentHashMap, String... topics) { + + if (topicConfigConcurrentHashMap == null) { + topicConfigConcurrentHashMap = new ConcurrentHashMap<>(); + + for (final String topic : topics) { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(topic); + topicConfig.setPerm(6); + topicConfig.setReadQueueNums(8); + topicConfig.setOrder(false); + topicConfigConcurrentHashMap.put(topic, topicConfig); + } + } + + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setDataVersion(brokerInfo.dataVersion); + topicConfigSerializeWrapper.setTopicConfigTable(topicConfigConcurrentHashMap); + + RegisterBrokerResult registerBrokerResult = routeInfoManager.registerBroker( + brokerInfo.clusterName, + brokerInfo.brokerAddr, + brokerInfo.brokerName, + brokerInfo.brokerId, + brokerInfo.haAddr, + null, + brokerInfo.enableActingMaster, + topicConfigSerializeWrapper, new ArrayList(), channel); + return registerBrokerResult; + } + + static class BrokerBasicInfo { + String clusterName; + String brokerName; + String brokerAddr; + String haAddr; + int brokerId; + boolean enableActingMaster; + + DataVersion dataVersion; + + static BrokerBasicInfo defaultBroker() { + BrokerBasicInfo basicInfo = new BrokerBasicInfo(); + DataVersion dataVersion = new DataVersion(); + dataVersion.setCounter(new AtomicLong(1)); + dataVersion.setTimestamp(System.currentTimeMillis()); + basicInfo.dataVersion = dataVersion; + + return basicInfo.id(0) + .name(DEFAULT_BROKER) + .cluster(DEFAULT_CLUSTER) + .addr(DEFAULT_ADDR) + .haAddr(DEFAULT_ADDR_PREFIX + "20911") + .enableActingMaster(true); + } + + UnRegisterBrokerRequestHeader unRegisterRequest() { + UnRegisterBrokerRequestHeader unRegisterBrokerRequest = new UnRegisterBrokerRequestHeader(); + unRegisterBrokerRequest.setBrokerAddr(brokerAddr); + unRegisterBrokerRequest.setBrokerName(brokerName); + unRegisterBrokerRequest.setClusterName(clusterName); + unRegisterBrokerRequest.setBrokerId((long) brokerId); + return unRegisterBrokerRequest; + } + + static BrokerBasicInfo slaveBroker() { + final BrokerBasicInfo slaveBroker = defaultBroker(); + return slaveBroker + .id(1) + .addr(DEFAULT_ADDR_PREFIX + "30911") + .haAddr(DEFAULT_ADDR_PREFIX + "40911") + .enableActingMaster(true); + } + + BrokerBasicInfo name(String name) { + this.brokerName = name; + return this; + } + + BrokerBasicInfo cluster(String name) { + this.clusterName = name; + return this; + } + + BrokerBasicInfo addr(String addr) { + this.brokerAddr = addr; + return this; + } + + BrokerBasicInfo id(int id) { + this.brokerId = id; + return this; + } + + BrokerBasicInfo haAddr(String addr) { + this.haAddr = addr; + return this; + } + + BrokerBasicInfo enableActingMaster(boolean enableActingMaster) { + this.enableActingMaster = enableActingMaster; + return this; + } + } + +} \ No newline at end of file