From b6ff649291453d3dc2d702fbba1548d991b610ac Mon Sep 17 00:00:00 2001 From: dongeforever Date: Sat, 6 Nov 2021 11:54:21 +0800 Subject: [PATCH] feat(all):new feature for static topic --- .../rocketmq/broker/BrokerController.java | 168 ++- .../broker/BrokerPathConfigHelper.java | 4 + .../rebalance/RebalanceLockManager.java | 9 +- .../domain/LogicalQueuesInfoInBroker.java | 116 -- .../rocketmq/broker/out/BrokerOuterAPI.java | 57 +- .../AbstractSendMessageProcessor.java | 174 +-- .../processor/AdminBrokerProcessor.java | 1013 +++++--------- .../processor/ConsumerManageProcessor.java | 101 +- .../processor/PullMessageProcessor.java | 369 ++++-- .../processor/SendMessageProcessor.java | 101 +- .../broker/slave/SlaveSynchronize.java | 23 +- .../broker/topic/TopicConfigManager.java | 115 +- .../topic/TopicQueueMappingCleanService.java | 332 +++++ .../topic/TopicQueueMappingManager.java | 259 ++++ .../rocketmq/broker/BrokerOuterAPITest.java | 2 +- .../rocketmq/broker/BrokerStartupTest.java | 20 +- .../processor/AdminBrokerProcessorTest.java | 298 +---- .../processor/PullMessageProcessorTest.java | 111 +- .../processor/SendMessageProcessorTest.java | 73 +- .../broker/topic/TopicConfigManagerTest.java | 138 -- .../topic/TopicQueueMappingManagerTest.java | 112 ++ .../consumer/DefaultMQPushConsumer.java | 2 +- .../rocketmq/client/consumer/PullResult.java | 1 + .../store/RemoteBrokerOffsetStore.java | 38 +- .../exception/OffsetNotFoundException.java | 20 +- .../rocketmq/client/impl/MQAdminImpl.java | 230 +--- .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +--- .../ConsumeMessageConcurrentlyService.java | 2 +- .../consumer/DefaultLitePullConsumerImpl.java | 42 +- .../consumer/DefaultMQPullConsumerImpl.java | 10 +- .../consumer/DefaultMQPushConsumerImpl.java | 87 +- .../client/impl/consumer/PullAPIWrapper.java | 339 +---- .../client/impl/consumer/PullResultExt.java | 12 + .../consumer/PullResultWithLogicalQueues.java | 96 -- .../client/impl/consumer/RebalanceImpl.java | 7 +- .../impl/consumer/RebalancePushImpl.java | 15 +- .../client/impl/factory/MQClientInstance.java | 169 +-- .../impl/producer/DefaultMQProducerImpl.java | 236 +--- .../producer/SendResultForLogicalQueue.java | 46 - ...DefaultMQPullConsumerLogicalQueueTest.java | 248 ---- .../store/RemoteBrokerOffsetStoreTest.java | 10 +- .../DefaultMQProducerLogicalQueueTest.java | 311 ----- .../producer/DefaultMQProducerTest.java | 28 +- .../DefaultMQProducerWithOpenTracingTest.java | 14 +- .../trace/DefaultMQProducerWithTraceTest.java | 26 +- ...nsactionMQProducerWithOpenTracingTest.java | 16 +- .../TransactionMQProducerWithTraceTest.java | 40 +- .../org/apache/rocketmq/common/MixAll.java | 20 +- .../apache/rocketmq/common/TopicConfig.java | 6 + .../rocketmq/common/admin/TopicOffset.java | 9 + .../rocketmq/common/protocol/RequestCode.java | 11 +- .../common/protocol/ResponseCode.java | 9 + ...essageQueueForLogicalQueueRequestBody.java | 50 - .../body/MigrateLogicalQueueBody.java | 42 - .../protocol/body/RegisterBrokerBody.java | 32 +- .../ReuseTopicLogicalQueueRequestBody.java | 59 - ...TopicConfigAndMappingSerializeWrapper.java | 68 + .../body/TopicConfigSerializeWrapper.java | 11 - ...=> TopicQueueMappingSerializeWrapper.java} | 36 +- .../header/CreateTopicRequestHeader.java | 12 + .../GetEarliestMsgStoretimeRequestHeader.java | 8 +- .../header/GetMaxOffsetRequestHeader.java | 25 +- .../header/GetMinOffsetRequestHeader.java | 8 +- .../header/GetTopicConfigRequestHeader.java | 5 +- .../GetTopicStatsInfoRequestHeader.java | 4 +- .../header/PullMessageRequestHeader.java | 8 +- .../header/PullMessageResponseHeader.java | 11 + .../QueryConsumerOffsetRequestHeader.java | 18 +- .../header/SearchOffsetRequestHeader.java | 8 +- .../header/SendMessageRequestHeader.java | 8 +- .../UpdateConsumerOffsetRequestHeader.java | 8 +- .../namesrv/GetRouteInfoRequestHeader.java | 20 - .../protocol/route/LogicalQueueRouteData.java | 309 ----- .../protocol/route/LogicalQueuesInfo.java | 87 -- .../route/LogicalQueuesInfoUnordered.java | 108 -- .../common/protocol/route/TopicRouteData.java | 28 +- .../protocol/route/TopicRouteDataNameSrv.java | 64 - .../rocketmq/common/rpc/ClientMetadata.java | 176 +++ .../rocketmq/common/rpc/RequestBuilder.java | 81 ++ .../apache/rocketmq/common/rpc/RpcClient.java | 35 + .../rocketmq/common/rpc/RpcClientHook.java | 27 + .../rocketmq/common/rpc/RpcClientImpl.java | 317 +++++ .../rocketmq/common/rpc/RpcClientUtils.java | 58 + .../RpcException.java} | 37 +- .../RpcRequest.java} | 30 +- .../rocketmq/common/rpc/RpcRequestHeader.java | 62 + .../rocketmq/common/rpc/RpcResponse.java | 70 + .../common/rpc/TopicQueueRequestHeader.java | 24 + .../common/rpc/TopicRequestHeader.java | 32 + .../statictopic/LogicQueueMappingItem.java | 212 +++ .../TopicConfigAndQueueMapping.java | 63 + .../statictopic/TopicQueueMappingContext.java | 99 ++ .../statictopic/TopicQueueMappingDetail.java | 144 ++ .../statictopic/TopicQueueMappingInfo.java | 161 +++ .../statictopic/TopicQueueMappingOne.java | 88 ++ .../statictopic/TopicQueueMappingUtils.java | 694 ++++++++++ .../TopicRemappingDetailWrapper.java | 104 ++ .../common/sysflag/MessageSysFlag.java | 1 - .../rocketmq/common/sysflag/PullSysFlag.java | 4 + .../rocketmq/common/ConfigManagerTest.java | 5 +- .../common/RegisterBrokerBodyTest.java | 3 +- .../apache/rocketmq/common/UtilAllTest.java | 3 + .../GenericMapSuperclassDeserializerTest.java | 57 - .../statictopic/TopicQueueMappingTest.java | 63 + .../TopicQueueMappingUtilsTest.java | 303 +++++ .../RocketMQ_Static_Topic_Logic_Queue_设计.md | 498 +++++++ .../statictopic/The_Scope_Of_Static_Topic.md | 116 ++ .../ClusterTestRequestProcessor.java | 2 +- .../processor/DefaultRequestProcessor.java | 19 +- .../namesrv/routeinfo/RouteInfoManager.java | 76 +- .../DefaultRequestProcessorTest.java | 21 +- .../remoting/netty/NettyRemotingClient.java | 1 + .../remoting/protocol/RemotingCommand.java | 53 +- .../protocol/RemotingCommandTest.java | 101 +- .../test/client/rmq/RMQNormalConsumer.java | 5 + .../test/client/rmq/RMQNormalProducer.java | 7 + .../clientinterface/AbstractMQProducer.java | 1 + .../apache/rocketmq/test/util/MQAdmin.java | 166 --- .../rocketmq/test/util/MQAdminTestUtils.java | 274 ++++ .../apache/rocketmq/test/base/BaseConf.java | 38 +- .../test/base/IntegrationTestBase.java | 7 +- .../rocketmq/test/smoke/LogicalQueueIT.java | 1170 ----------------- .../test/statictopic/StaticTopicIT.java | 465 +++++++ .../tools/admin/DefaultMQAdminExt.java | 75 +- .../tools/admin/DefaultMQAdminExtImpl.java | 126 +- .../rocketmq/tools/admin/MQAdminExt.java | 43 +- .../rocketmq/tools/admin/MQAdminUtils.java | 343 +++++ .../tools/command/MQAdminStartup.java | 24 +- ...DeleteTopicLogicalQueueMappingCommand.java | 91 -- .../MigrateTopicLogicalQueueCommand.java | 210 --- .../QueryTopicLogicalQueueMappingCommand.java | 123 -- ...UpdateTopicLogicalQueueMappingCommand.java | 159 --- .../UpdateTopicLogicalQueueNumCommand.java | 285 ---- .../topic/RemappingStaticTopicSubCommand.java | 207 +++ .../topic/UpdateStaticTopicSubCommand.java | 208 +++ .../command/topic/UpdateTopicSubCommand.java | 20 - .../tools/admin/DefaultMQAdminExtTest.java | 31 +- .../message/ConsumeMessageCommandTest.java | 4 +- 138 files changed, 7598 insertions(+), 7295 deletions(-) delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java delete mode 100644 broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java rename common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java => client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java (59%) delete mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java delete mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java delete mode 100644 client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java delete mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigAndMappingSerializeWrapper.java rename common/src/main/java/org/apache/rocketmq/common/protocol/body/{UpdateTopicLogicalQueueMappingRequestBody.java => TopicQueueMappingSerializeWrapper.java} (53%) delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java delete mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/ClientMetadata.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RequestBuilder.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcClient.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientHook.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientImpl.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientUtils.java rename common/src/main/java/org/apache/rocketmq/common/{protocol/body/SealTopicLogicalQueueRequestBody.java => rpc/RpcException.java} (53%) rename common/src/main/java/org/apache/rocketmq/common/{protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java => rpc/RpcRequest.java} (59%) create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/RpcResponse.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/TopicQueueRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/rpc/TopicRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/LogicQueueMappingItem.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicConfigAndQueueMapping.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingContext.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingDetail.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingInfo.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingOne.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtils.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/statictopic/TopicRemappingDetailWrapper.java delete mode 100644 common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java create mode 100644 common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtilsTest.java create mode 100644 docs/cn/statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md create mode 100644 docs/cn/statictopic/The_Scope_Of_Static_Topic.md delete mode 100644 test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java create mode 100644 test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java delete mode 100644 test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java create mode 100644 test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminUtils.java delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 5eb916922f..76c8007f63 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -17,29 +17,6 @@ package org.apache.rocketmq.broker; import com.google.common.collect.Maps; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.AbstractMap; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.broker.client.ClientHousekeepingService; import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; @@ -49,7 +26,6 @@ import org.apache.rocketmq.broker.client.ProducerManager; import org.apache.rocketmq.broker.client.net.Broker2Client; import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager; import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filtersrv.FilterServerManager; @@ -80,6 +56,8 @@ import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService; +import org.apache.rocketmq.broker.topic.TopicQueueMappingManager; import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener; import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; @@ -87,11 +65,9 @@ import org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageC import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl; import org.apache.rocketmq.broker.util.ServiceProvider; -import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.Configuration; import org.apache.rocketmq.common.DataVersion; -import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; @@ -99,18 +75,16 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.common.stats.MomentStatsItem; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.RemotingServer; import org.apache.rocketmq.remoting.common.TlsMode; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingServer; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; @@ -128,6 +102,30 @@ import org.apache.rocketmq.store.dledger.DLedgerCommitLog; import org.apache.rocketmq.store.stats.BrokerStats; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; + public class BrokerController { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private static final InternalLogger LOG_PROTECTION = InternalLoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); @@ -179,6 +177,7 @@ public class BrokerController { private RemotingServer remotingServer; private RemotingServer fastRemotingServer; private TopicConfigManager topicConfigManager; + private TopicQueueMappingManager topicQueueMappingManager; private ExecutorService sendMessageExecutor; private ExecutorService pullMessageExecutor; private ExecutorService ackMessageExecutor; @@ -195,12 +194,13 @@ public class BrokerController { private InetSocketAddress storeHost; private BrokerFastFailure brokerFastFailure; private Configuration configuration; + private TopicQueueMappingCleanService topicQueueMappingCleanService; private FileWatchService fileWatchService; private TransactionalMessageCheckService transactionalMessageCheckService; private TransactionalMessageService transactionalMessageService; private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; private Future slaveSyncFuture; - private Map accessValidatorMap = new HashMap(); + private Map accessValidatorMap = new HashMap(); private long shouldStartTime; public BrokerController( @@ -215,6 +215,7 @@ public class BrokerController { this.messageStoreConfig = messageStoreConfig; this.consumerOffsetManager = new ConsumerOffsetManager(this); this.topicConfigManager = new TopicConfigManager(this); + this.topicQueueMappingManager = new TopicQueueMappingManager(this); this.pullMessageProcessor = new PullMessageProcessor(this); this.pullRequestHoldService = new PullRequestHoldService(this); this.popMessageProcessor = new PopMessageProcessor(this); @@ -231,7 +232,7 @@ public class BrokerController { this.clientHousekeepingService = new ClientHousekeepingService(this); this.broker2Client = new Broker2Client(this); this.subscriptionGroupManager = new SubscriptionGroupManager(this); - this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig); + this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, this); this.filterServerManager = new FilterServerManager(this); this.assignmentManager = new AssignmentManager(this); @@ -287,18 +288,20 @@ public class BrokerController { public boolean initialize() throws CloneNotSupportedException { boolean result = this.topicConfigManager.load(); + result = result && this.topicQueueMappingManager.load(); + result = result && this.consumerOffsetManager.load(); result = result && this.subscriptionGroupManager.load(); result = result && this.consumerFilterManager.load(); if (result) { try { - DefaultMessageStore messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig); - messageStore.registerCleanFileHook(topicConfigManager.getLogicalQueueCleanHook()); - this.messageStore = messageStore; + this.messageStore = + new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, + this.brokerConfig); if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); - ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); + ((DLedgerCommitLog) ((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin @@ -342,7 +345,6 @@ public class BrokerController { this.ackThreadPoolQueue, new ThreadFactoryImpl("AckMessageThread_")); - this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( this.brokerConfig.getProcessReplyMessageThreadPoolNums(), this.brokerConfig.getProcessReplyMessageThreadPoolNums(), @@ -485,11 +487,11 @@ public class BrokerController { this.scheduledExecutorService.scheduleAtFixedRate(() -> { try { - BrokerController.this.refreshBrokerNameMapping(); + BrokerController.this.brokerOuterAPI.refreshMetadata(); } catch (Exception e) { - log.error("ScheduledTask examineBrokerClusterInfo exception", e); + log.error("ScheduledTask refresh metadata exception", e); } - }, 10, 10, TimeUnit.SECONDS); + }, 1, 5, TimeUnit.SECONDS); if (!messageStoreConfig.isEnableDLegerCommitLog()) { if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { @@ -513,6 +515,8 @@ public class BrokerController { } } + this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this); + if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) { // Register a listener to reload SslContext try { @@ -587,9 +591,9 @@ public class BrokerController { return; } - for (AccessValidator accessValidator: accessValidators) { + for (AccessValidator accessValidator : accessValidators) { final AccessValidator validator = accessValidator; - accessValidatorMap.put(validator.getClass(),validator); + accessValidatorMap.put(validator.getClass(), validator); this.registerServerRPCHook(new RPCHook() { @Override @@ -605,26 +609,17 @@ public class BrokerController { } } - private void initialRpcHooks() { List rpcHooks = ServiceProvider.load(ServiceProvider.RPC_HOOK_ID, RPCHook.class); if (rpcHooks == null || rpcHooks.isEmpty()) { return; } - for (RPCHook rpcHook: rpcHooks) { + for (RPCHook rpcHook : rpcHooks) { this.registerServerRPCHook(rpcHook); } } - private void refreshBrokerNameMapping() throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - ClusterInfo brokerClusterInfo = this.brokerOuterAPI.getBrokerClusterInfo(); - brokerClusterInfo.getBrokerAddrTable().forEach((brokerName, data) -> { - String masterBrokerAddr = data.getBrokerAddrs().get(MixAll.MASTER_ID); - this.brokerName2AddrMap.put(brokerName, masterBrokerAddr); - }); - } - public String getBrokerAddrByName(String brokerName) { return this.brokerName2AddrMap.get(brokerName); } @@ -878,6 +873,10 @@ public class BrokerController { this.fastRemotingServer.shutdown(); } + if (this.topicQueueMappingCleanService != null) { + this.topicQueueMappingCleanService.shutdown(); + } + if (this.fileWatchService != null) { this.fileWatchService.shutdown(); } @@ -995,6 +994,10 @@ public class BrokerController { this.fastRemotingServer.start(); } + if (this.topicQueueMappingCleanService != null) { + this.topicQueueMappingCleanService.start(); + } + if (this.fileWatchService != null) { this.fileWatchService.start(); } @@ -1041,7 +1044,6 @@ public class BrokerController { this.brokerFastFailure.start(); } - } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { @@ -1053,7 +1055,7 @@ public class BrokerController { return; } - TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper(); topicConfigSerializeWrapper.setDataVersion(dataVersion); ConcurrentMap topicConfigTable = topicConfigList.stream() @@ -1074,49 +1076,41 @@ public class BrokerController { .collect(Collectors.toConcurrentMap(TopicConfig::getTopicName, Function.identity())); topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable); - String brokerName = this.brokerConfig.getBrokerName(); - Map logicalQueuesInfoMap = topicConfigList.stream() + Map topicQueueMappingInfoMap = topicConfigList.stream() .map(TopicConfig::getTopicName) - .map(topicName -> Optional.ofNullable(this.topicConfigManager.selectLogicalQueuesInfo(topicName)) - .map(info -> { - info.readLock().lock(); - try { - return new AbstractMap.SimpleImmutableEntry<>(topicName, new LogicalQueuesInfoInBroker(info, data -> Objects.equals(data.getBrokerName(), brokerName))); - } finally { - info.readLock().unlock(); - } - }) + .map(topicName -> Optional.ofNullable(this.topicQueueMappingManager.getTopicQueueMapping(topicName)) + .map(info -> new AbstractMap.SimpleImmutableEntry<>(topicName, TopicQueueMappingDetail.cloneAsMappingInfo(info))) .orElse(null)) .filter(Objects::nonNull) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - if (!logicalQueuesInfoMap.isEmpty()) { - topicConfigSerializeWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap); + if (!topicQueueMappingInfoMap.isEmpty()) { + topicConfigSerializeWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); } doRegisterBrokerAll(true, false, topicConfigSerializeWrapper); } public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { - TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); + + TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); + + topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion()); + topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable()); + + topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map( + entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())) + ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { ConcurrentHashMap topicConfigTable = new ConcurrentHashMap(); - Map logicalQueuesInfoMap = Maps.newHashMapWithExpectedSize(topicConfigWrapper.getTopicConfigTable().size()); for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { - String topicName = topicConfig.getTopicName(); TopicConfig tmp = - new TopicConfig(topicName, topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), + new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), this.brokerConfig.getBrokerPermission()); - topicConfigTable.put(topicName, tmp); - LogicalQueuesInfoInBroker logicalQueuesInfo = this.topicConfigManager.selectLogicalQueuesInfo(topicName); - if (logicalQueuesInfo != null) { - String brokerName = this.brokerConfig.getBrokerName(); - logicalQueuesInfoMap.put(topicName, new LogicalQueuesInfoInBroker(logicalQueuesInfo, data -> Objects.equals(data.getBrokerName(), brokerName))); - } + topicConfigTable.put(topicConfig.getTopicName(), tmp); } topicConfigWrapper.setTopicConfigTable(topicConfigTable); - topicConfigWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap); } if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), @@ -1129,7 +1123,7 @@ public class BrokerController { } private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway, - TopicConfigSerializeWrapper topicConfigWrapper) { + TopicConfigAndMappingSerializeWrapper topicConfigWrapper) { List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( this.brokerConfig.getBrokerClusterName(), this.getBrokerAddr(), @@ -1184,6 +1178,10 @@ public class BrokerController { this.topicConfigManager = topicConfigManager; } + public TopicQueueMappingManager getTopicQueueMappingManager() { + return topicQueueMappingManager; + } + public String getHAServerAddr() { return this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort(); } @@ -1297,7 +1295,6 @@ public class BrokerController { this.transactionalMessageCheckListener = transactionalMessageCheckListener; } - public BlockingQueue getEndTransactionThreadPoolQueue() { return endTransactionThreadPoolQueue; @@ -1318,8 +1315,7 @@ public class BrokerController { public void run() { try { BrokerController.this.slaveSynchronize.syncAll(); - } - catch (Throwable e) { + } catch (Throwable e) { log.error("ScheduledTask SlaveSynchronize syncAll error.", e); } } @@ -1365,8 +1361,6 @@ public class BrokerController { log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId); } - - public void changeToMaster(BrokerRole role) { if (role == BrokerRole.SLAVE) { return; @@ -1435,4 +1429,8 @@ public class BrokerController { public QueryAssignmentProcessor getQueryAssignmentProcessor() { return queryAssignmentProcessor; } + + public TopicQueueMappingCleanService getTopicQueueMappingCleanService() { + return topicQueueMappingCleanService; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java index 43a9946f71..6360d54bf3 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java @@ -35,6 +35,10 @@ public class BrokerPathConfigHelper { return rootDir + File.separator + "config" + File.separator + "topics.json"; } + public static String getTopicQueueMappingPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "topicQueueMapping.json"; + } + public static String getConsumerOffsetPath(final String rootDir) { return rootDir + File.separator + "config" + File.separator + "consumerOffset.json"; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java index 678b1f54a1..9056998cd8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java @@ -16,16 +16,17 @@ */ package org.apache.rocketmq.broker.client.rebalance; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.common.message.MessageQueue; public class RebalanceLockManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java b/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java deleted file mode 100644 index a6728c828b..0000000000 --- a/broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.broker.domain; - -import com.alibaba.fastjson.parser.ParserConfig; -import com.google.common.collect.Maps; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; - -import static java.util.Optional.ofNullable; - -public class LogicalQueuesInfoInBroker extends LogicalQueuesInfo { - private final ConcurrentMap> queueId2LogicalQueueMap = Maps.newConcurrentMap(); - - public LogicalQueuesInfoInBroker() { - } - - public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other) { - this(other, null); - } - - // deep copy - public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other, Predicate predicate) { - other.readLock().lock(); - try { - for (Entry> entry : other.entrySet()) { - Stream stream = entry.getValue().stream(); - if (predicate != null) { - stream = stream.filter(predicate); - } - this.put(entry.getKey(), stream.map(LogicalQueueRouteData::new).collect(Collectors.toList())); - } - } finally { - other.readLock().unlock(); - } - } - - public void updateQueueRouteDataByQueueId(int queueId, LogicalQueueRouteData queueRouteData) { - if (queueRouteData == null) { - return; - } - ConcurrentHashMapUtil.computeIfAbsent(queueId2LogicalQueueMap, queueId, k -> new ConcurrentSkipListMap<>()).put(queueRouteData.getOffsetDelta(), queueRouteData); - } - - /** - * find logical queue route data for message queues owned by this broker - */ - public LogicalQueueRouteData queryQueueRouteDataByQueueId(int queueId, long offset) { - ConcurrentNavigableMap m = this.queueId2LogicalQueueMap.get(queueId); - if (m == null || m.isEmpty()) { - return null; - } - Entry entry = m.floorEntry(offset); - if (entry == null) { - return null; - } - return entry.getValue(); - } - - public void deleteQueueRouteData(LogicalQueueRouteData logicalQueueRouteData) { - ConcurrentNavigableMap m = this.queueId2LogicalQueueMap.get(logicalQueueRouteData.getQueueId()); - if (m != null) { - m.remove(logicalQueueRouteData.getOffsetDelta(), logicalQueueRouteData); - } - } - - public LogicalQueueRouteData nextAvailableLogicalRouteData(LogicalQueueRouteData queueRouteData, - Predicate predicate) { - this.readLock().lock(); - try { - List queueRouteDataList = ofNullable(this.get(queueRouteData.getLogicalQueueIndex())).orElse(Collections.emptyList()); - int idx = Collections.binarySearch(queueRouteDataList, queueRouteData); - if (idx >= 0) { - for (int i = idx + 1, size = queueRouteDataList.size(); i < size; i++) { - LogicalQueueRouteData tmp = queueRouteDataList.get(i); - if (predicate.test(tmp)) { - return tmp; - } - } - } - } finally { - this.readLock().unlock(); - } - return null; - } - - static { - // workaround https://github.com/alibaba/fastjson/issues/3730 - ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoInBroker.class, GenericMapSuperclassDeserializer.INSTANCE); - } -} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java index 4d33663bb9..8a2093a621 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java @@ -16,13 +16,7 @@ */ package org.apache.rocketmq.broker.out; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.common.DataVersion; @@ -39,6 +33,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader; @@ -47,6 +42,9 @@ import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestH import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.rpc.RpcClient; +import org.apache.rocketmq.common.rpc.RpcClientImpl; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.InvokeCallback; @@ -62,21 +60,39 @@ import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + public class BrokerOuterAPI { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final RemotingClient remotingClient; + private final BrokerController brokerController; private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr()); private String nameSrvAddr = null; + private final String currBrokerName; private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, new ArrayBlockingQueue(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); - public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { - this(nettyClientConfig, null); + private ClientMetadata clientMetadata; + private RpcClient rpcClient; + + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, final BrokerController brokerController) { + + this(nettyClientConfig, null, brokerController, new ClientMetadata()); } - public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) { + private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, final BrokerController brokerController, ClientMetadata clientMetadata) { this.remotingClient = new NettyRemotingClient(nettyClientConfig); + this.clientMetadata = clientMetadata; this.remotingClient.registerRPCHook(rpcHook); + this.brokerController = brokerController; + this.currBrokerName = brokerController.getBrokerConfig().getBrokerName(); + this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient); } public void start() { @@ -140,7 +156,7 @@ public class BrokerOuterAPI { requestHeader.setCompressed(compressed); RegisterBrokerBody requestBody = new RegisterBrokerBody(); - requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); + requestBody.setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper)); requestBody.setFilterServerList(filterServerList); final byte[] body = requestBody.encode(compressed); final int bodyCrc32 = UtilAll.crc32(body); @@ -327,7 +343,7 @@ public class BrokerOuterAPI { return changedList; } - public TopicConfigSerializeWrapper getAllTopicConfig( + public TopicConfigAndMappingSerializeWrapper getAllTopicConfig( final String addr) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null); @@ -336,7 +352,7 @@ public class BrokerOuterAPI { assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class); + return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigAndMappingSerializeWrapper.class); } default: break; @@ -453,4 +469,19 @@ public class BrokerOuterAPI { public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException { this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback); } + + + public void refreshMetadata() throws Exception { + ClusterInfo brokerClusterInfo = getBrokerClusterInfo(); + clientMetadata.refreshClusterInfo(brokerClusterInfo); + } + + + public ClientMetadata getClientMetadata() { + return clientMetadata; + } + + public RpcClient getRpcClient() { + return rpcClient; + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index af14b5ba00..3303d70e4a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -16,31 +16,21 @@ */ package org.apache.rocketmq.broker.processor; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; import io.netty.channel.ChannelHandlerContext; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; - import java.util.concurrent.ThreadLocalRandom; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; -import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; -import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.constant.DBMsgConstants; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; @@ -54,23 +44,17 @@ import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.TopicSysFlag; -import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.ChannelUtil; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; import org.apache.rocketmq.store.MessageExtBrokerInner; -import org.apache.rocketmq.store.PutMessageResult; public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -80,8 +64,6 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc protected final SocketAddress storeHost; private List sendMessageHookList; - private final ConcurrentMap inFlyWritingCounterMap = Maps.newConcurrentMap(); - public AbstractSendMessageProcessor(final BrokerController brokerController) { this.brokerController = brokerController; this.storeHost = @@ -419,158 +401,4 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc public boolean rejectRequest() { return false; } - - - public ConcurrentMap getInFlyWritingCounterMap() { - return inFlyWritingCounterMap; - } - - protected LogicalQueueContext buildLogicalQueueContext(String topic, int queueId, - RemotingCommand response) { - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - return noopLogicalQueueContext; - } - // writable route data will has largest offset - LogicalQueueRouteData curQueueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, Long.MAX_VALUE); - if (curQueueRouteData == null) { - // topic enabled logical queue, but some message queues are not converted or being converted - String msg = String.format(Locale.ENGLISH, "queueId %d not included in logical queue", queueId); - log.debug("buildLogicalQueueContext unexpected error, topic {} {}", topic, msg); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(msg); - return noopLogicalQueueContext; - } - LongAdder inFlyWritingCounter = ConcurrentHashMapUtil.computeIfAbsent(inFlyWritingCounterMap, new TopicQueueId(topic, queueId), ignore -> new LongAdder()); - return new LogicalQueueContext(topic, queueId, logicalQueuesInfo, curQueueRouteData, inFlyWritingCounter); - } - - protected class LogicalQueueContext { - private final String topic; - private final int queueId; - private final LogicalQueuesInfoInBroker logicalQueuesInfo; - private final LogicalQueueRouteData curQueueRouteData; - private final LongAdder inFlyWritingCounter; - - public LogicalQueueContext(String topic, int queueId, - LogicalQueuesInfoInBroker logicalQueuesInfo, - LogicalQueueRouteData curQueueRouteData, LongAdder inFlyWritingCounter) { - this.topic = topic; - this.queueId = queueId; - this.logicalQueuesInfo = logicalQueuesInfo; - this.curQueueRouteData = curQueueRouteData; - this.inFlyWritingCounter = inFlyWritingCounter; - } - - public CompletableFuture hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader, - RemotingCommand request, RemotingCommand response) { - if (curQueueRouteData.isWritable()) { - this.inFlyWritingCounter.increment(); - return null; - } - int logicalQueueIdx = curQueueRouteData.getLogicalQueueIndex(); - List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); - LogicalQueueRouteData writableQueueRouteData = null; - for (int i = queueRouteDataList.size() - 1; i >= 0; i--) { - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); - if (queueRouteData.isWritable()) { - writableQueueRouteData = queueRouteData; - break; - } - } - if (writableQueueRouteData == null) { - response.setCode(ResponseCode.NO_PERMISSION); - response.setRemark(String.format(Locale.ENGLISH, "broker[%s] topic[%s] queueId[%d] logicalQueueIdx[%d] not writable", AbstractSendMessageProcessor.this.brokerController.getBrokerConfig().getBrokerIP1(), topic, queueId, logicalQueueIdx)); - return CompletableFuture.completedFuture(response); - } - if ((Optional.ofNullable(requestHeader.getSysFlag()).orElse(0) & MessageSysFlag.LOGICAL_QUEUE_FLAG) > 0) { - // new client, use redirect - response.setCode(ResponseCode.NO_PERMISSION); - response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); - response.setBody(RemotingSerializable.encode(ImmutableList.of(curQueueRouteData, writableQueueRouteData))); - return CompletableFuture.completedFuture(response); - } else { - // old client, use forward - this.logicalQueueHookForward(ctx, writableQueueRouteData, requestHeader, request, response); - } - if (response.getCode() != -1) { - return CompletableFuture.completedFuture(response); - } else if (response.getCode() == ResponseCode.ASYNC_AND_RETURN_NULL) { - return CompletableFuture.completedFuture(null); - } - return null; - } - - private void logicalQueueHookForward(ChannelHandlerContext ctx, - LogicalQueueRouteData writableQueueRouteData, - SendMessageRequestHeader requestHeader, RemotingCommand request, - RemotingCommand response) { - response.setCode(ResponseCode.SUCCESS); - requestHeader.setQueueId(writableQueueRouteData.getQueueId()); - request.writeCustomHeader(requestHeader); - String brokerName = writableQueueRouteData.getBrokerName(); - BrokerController brokerController = AbstractSendMessageProcessor.this.brokerController; - String brokerAddr = brokerController.getBrokerAddrByName(brokerName); - if (brokerAddr == null) { - log.warn("getBrokerAddrByName brokerName={} got null, fallback to queueRouteData.getBrokerAddr()", brokerName); - brokerAddr = writableQueueRouteData.getBrokerAddr(); - } - if (brokerAddr == null) { - response.setCode(ResponseCode.SYSTEM_ERROR); - String msg = String.format(Locale.ENGLISH, "unknown brokerName %s", brokerName); - response.setRemark(msg); - log.warn("logicalQueueHookForward can not look up brokerName={}: {}", brokerName, requestHeader); - return; - } - try { - String finalBrokerAddr = brokerAddr; - brokerController.getBrokerOuterAPI().forwardRequest(brokerAddr, request, brokerController.getBrokerConfig().getForwardTimeout(), responseFuture -> { - RemotingCommand forwardResponse = responseFuture.getResponseCommand(); - if (forwardResponse == null) { - forwardResponse = response; - forwardResponse.setCode(ResponseCode.SYSTEM_ERROR); - if (!responseFuture.isSendRequestOK()) { - forwardResponse.setRemark(String.format(Locale.ENGLISH, "send request failed to %s: %s", finalBrokerAddr, responseFuture.getCause())); - } else if (responseFuture.isTimeout()) { - forwardResponse.setRemark(String.format(Locale.ENGLISH, "wait response from %s timeout: %dms", finalBrokerAddr, responseFuture.getTimeoutMillis())); - } else { - forwardResponse.setRemark(String.format(Locale.ENGLISH, "unknown reason. addr: %s, timeoutMillis: %d: %s", finalBrokerAddr, responseFuture.getTimeoutMillis(), responseFuture.getCause())); - } - } else { - CommandCustomHeader customHeader = forwardResponse.readCustomHeader(); - if (customHeader instanceof SendMessageResponseHeader) { - SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) customHeader; - Integer forwardQueueId = responseHeader.getQueueId(); - forwardResponse.addExtField(MessageConst.PROPERTY_FORWARD_QUEUE_ID, forwardQueueId != null ? Integer.toString(forwardQueueId) : "null"); - responseHeader.setQueueId(requestHeader.getQueueId()); - // queueOffset should not be changed since forwarded broker will add delta to it. - } - } - AbstractSendMessageProcessor.this.doResponse(ctx, request, forwardResponse); - }); - response.setCode(ResponseCode.ASYNC_AND_RETURN_NULL); - } catch (Exception e) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("forward error"); - log.warn(String.format(Locale.ENGLISH, "logicalQueueHookForward to %s error", brokerAddr), e); - } - } - - public void hookAfterPut(CompletableFuture putMessageResult) { - Optional.ofNullable(putMessageResult).orElse(CompletableFuture.completedFuture(null)).whenComplete((result, throwable) -> { - this.inFlyWritingCounter.decrement(); - }); - } - } - - private final LogicalQueueContext noopLogicalQueueContext = new LogicalQueueContext(null, 0, null, null, null) { - @Override public CompletableFuture hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader, - RemotingCommand request, RemotingCommand response) { - return null; - } - - @Override public void hookAfterPut(CompletableFuture putMessageResult) { - } - }; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index e7b7949560..2891bafe98 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -18,48 +18,22 @@ package org.apache.rocketmq.broker.processor; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.io.UnsupportedEncodingException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Properties; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.LongAdder; -import java.util.stream.Collectors; import org.apache.rocketmq.acl.AccessValidator; import org.apache.rocketmq.acl.plain.PlainAccessValidator; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; -import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; import org.apache.rocketmq.common.AclConfig; +import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.PlainAccessConfig; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.TopicQueueId; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.admin.ConsumeStats; import org.apache.rocketmq.common.admin.OffsetWrapper; @@ -80,29 +54,24 @@ import org.apache.rocketmq.common.protocol.body.Connection; import org.apache.rocketmq.common.protocol.body.ConsumeQueueData; import org.apache.rocketmq.common.protocol.body.ConsumeStatsList; import org.apache.rocketmq.common.protocol.body.ConsumerConnection; -import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody; import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; -import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; -import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; -import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.CloneGroupOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; -import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.GetAllTopicConfigResponseHeader; import org.apache.rocketmq.common.protocol.header.GetBrokerAclConfigResponseHeader; @@ -127,7 +96,6 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumeQueueRequestHeader import org.apache.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; -import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; @@ -137,8 +105,16 @@ import org.apache.rocketmq.common.protocol.header.ViewBrokerStatsDataRequestHead import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; import org.apache.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.rpc.RpcClient; +import org.apache.rocketmq.common.rpc.RpcClientUtils; +import org.apache.rocketmq.common.rpc.RpcException; +import org.apache.rocketmq.common.rpc.RpcRequest; +import org.apache.rocketmq.common.rpc.RpcResponse; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; import org.apache.rocketmq.common.stats.StatsItem; import org.apache.rocketmq.common.stats.StatsSnapshot; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; @@ -155,7 +131,6 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; -import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.ConsumeQueueExt; import org.apache.rocketmq.store.DefaultMessageStore; @@ -166,12 +141,31 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import java.io.UnsupportedEncodingException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentMap; + +import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; + public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; + private final RpcClient rpcClient; + private final BrokerConfig brokerConfig; public AdminBrokerProcessor(final BrokerController brokerController) { this.brokerController = brokerController; + this.brokerConfig = brokerController.getBrokerConfig(); + this.rpcClient = brokerController.getBrokerOuterAPI().getRpcClient(); } @Override @@ -264,24 +258,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return getBrokerClusterAclConfig(ctx, request); case RequestCode.GET_TOPIC_CONFIG: return getTopicConfig(ctx, request); - case RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING: - return updateTopicLogicalQueueMapping(ctx, request); - case RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING: - return deleteTopicLogicalQueueMapping(ctx, request); - case RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING: - return queryTopicLogicalQueueMapping(ctx, request); - case RequestCode.SEAL_TOPIC_LOGICAL_QUEUE: - return sealTopicLogicalQueue(ctx, request); - case RequestCode.REUSE_TOPIC_LOGICAL_QUEUE: - return reuseTopicLogicalQueue(ctx, request); - case RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE: - return createMessageQueueForLogicalQueue(ctx, request); - case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE: - return migrateTopicLogicalQueuePrepare(ctx, request); - case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT: - return migrateTopicLogicalQueueCommit(ctx, request); - case RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY: - return migrateTopicLogicalQueueNotify(ctx, request); + case RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC: + return this.updateAndCreateStaticTopic(ctx, request); default: return getUnknownCmdResponse(ctx, request); } @@ -323,6 +301,51 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } + private synchronized RemotingCommand updateAndCreateStaticTopic(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final CreateTopicRequestHeader requestHeader = + (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); + log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + final TopicQueueMappingDetail topicQueueMappingDetail = RemotingSerializable.decode(request.getBody(), TopicQueueMappingDetail.class); + + String topic = requestHeader.getTopic(); + + if (!TopicValidator.validateTopic(topic, response)) { + return response; + } + if (TopicValidator.isSystemTopic(topic, response)) { + return response; + } + boolean force = false; + if (requestHeader.getForce() != null && requestHeader.getForce()) { + force = true; + } + + TopicConfig topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); + topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); + topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); + topicConfig.setPerm(requestHeader.getPerm()); + topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader.getTopicSysFlag()); + + try { + this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); + + this.brokerController.getTopicQueueMappingManager().updateTopicQueueMapping(topicQueueMappingDetail, force, false, true); + + this.brokerController.registerIncrementBrokerData(topicConfig, this.brokerController.getTopicConfigManager().getDataVersion()); + response.setCode(ResponseCode.SUCCESS); + } catch (Exception e) { + log.error("Update static topic failed for [{}]", request, e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); + } + return response; + } + + private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); @@ -340,12 +363,14 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements } this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); + this.brokerController.getTopicQueueMappingManager().delete(topic); + this.brokerController.getMessageStore() .cleanUnusedTopic(this.brokerController.getTopicConfigManager().getTopicConfigTable().keySet()); if (this.brokerController.getBrokerConfig().isAutoDeleteUnusedStats()) { this.brokerController.getBrokerStatsManager().onTopicDeleted(requestHeader.getTopic()); } - this.brokerController.getTopicConfigManager().deleteQueueRouteData(requestHeader.getTopic()); + response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -518,7 +543,16 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements // final GetAllTopicConfigResponseHeader responseHeader = // (GetAllTopicConfigResponseHeader) response.readCustomHeader(); - String content = this.brokerController.getTopicConfigManager().encode(); + TopicConfigAndMappingSerializeWrapper topicConfigAndMappingSerializeWrapper = new TopicConfigAndMappingSerializeWrapper(); + + topicConfigAndMappingSerializeWrapper.setDataVersion(this.brokerController.getTopicConfigManager().getDataVersion()); + topicConfigAndMappingSerializeWrapper.setTopicConfigTable(this.brokerController.getTopicConfigManager().getTopicConfigTable()); + + topicConfigAndMappingSerializeWrapper.setMappingDataVersion(this.brokerController.getTopicQueueMappingManager().getDataVersion()); + topicConfigAndMappingSerializeWrapper.setTopicQueueMappingDetailMap(this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable()); + + + String content = topicConfigAndMappingSerializeWrapper.toJson(); if (content != null && content.length() > 0) { try { response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); @@ -603,6 +637,62 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } + private RemotingCommand rewriteRequestForStaticTopic(SearchOffsetRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + List mappingItems = mappingContext.getMappingItemList(); + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname())); + } + //TO DO should make sure the timestampOfOffset is equal or bigger than the searched timestamp + Long timestamp = requestHeader.getTimestamp(); + long offset = -1; + for (int i = 0; i < mappingItems.size(); i++) { + LogicQueueMappingItem item = mappingItems.get(i); + if (!item.checkIfLogicoffsetDecided()) { + continue; + } + if (mappingDetail.getBname().equals(item.getBname())) { + offset = this.brokerController.getMessageStore().getOffsetInQueueByTime(mappingContext.getTopic(), item.getQueueId(), timestamp); + if (offset > 0) { + offset = item.computeStaticQueueOffsetStrictly(offset); + break; + } + } else { + requestHeader.setLo(false); + requestHeader.setTimestamp(timestamp); + requestHeader.setQueueId(item.getQueueId()); + requestHeader.setBname(item.getBname()); + RpcRequest rpcRequest = new RpcRequest(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader, null); + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + SearchOffsetResponseHeader offsetResponseHeader = (SearchOffsetResponseHeader) rpcResponse.getHeader(); + if (offsetResponseHeader.getOffset() < 0 + || (item.checkIfEndOffsetDecided() && offsetResponseHeader.getOffset() >= item.getEndOffset())) { + continue; + } else { + offset = item.computeStaticQueueOffsetStrictly(offsetResponseHeader.getOffset()); + } + + } + } + final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + final SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + private RemotingCommand searchOffsetByTimestamp(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); @@ -610,6 +700,13 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements final SearchOffsetRequestHeader requestHeader = (SearchOffsetRequestHeader) request.decodeCommandCustomHeader(SearchOffsetRequestHeader.class); + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); + + RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + long offset = this.brokerController.getMessageStore().getOffsetInQueueByTime(requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getTimestamp()); @@ -620,6 +717,50 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } + private RemotingCommand rewriteRequestForStaticTopic(GetMaxOffsetRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + if (mappingContext.getMappingDetail() == null) { + return null; + } + + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem(); + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname())); + } + + try { + LogicQueueMappingItem maxItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), Long.MAX_VALUE, true); + assert maxItem != null; + assert maxItem.getLogicOffset() >= 0; + requestHeader.setBname(maxItem.getBname()); + requestHeader.setLo(false); + requestHeader.setQueueId(mappingItem.getQueueId()); + + long maxPhysicalOffset = Long.MAX_VALUE; + if (maxItem.getBname().equals(mappingDetail.getBname())) { + //current broker + maxPhysicalOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(mappingContext.getTopic(), mappingItem.getQueueId()); + } else { + RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_MAX_OFFSET, requestHeader, null); + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + GetMaxOffsetResponseHeader offsetResponseHeader = (GetMaxOffsetResponseHeader) rpcResponse.getHeader(); + maxPhysicalOffset = offsetResponseHeader.getOffset(); + } + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); + final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(maxItem.computeStaticQueueOffsetStrictly(maxPhysicalOffset)); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); @@ -627,33 +768,13 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements final GetMaxOffsetRequestHeader requestHeader = (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); - String topic = requestHeader.getTopic(); - int queueId = requestHeader.getQueueId(); - - if (requestHeader.getLogicalQueue()) { - LogicalQueuesInfoInBroker logicalQueuesInfo = this.brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo != null) { - // max offset must be in the queue route with largest offset - LogicalQueueRouteData requestLogicalQueueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, Long.MAX_VALUE); - if (requestLogicalQueueRouteData != null) { - logicalQueuesInfo.readLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(requestLogicalQueueRouteData.getLogicalQueueIndex()); - if (queueRouteDataList != null && !queueRouteDataList.isEmpty()) { - LogicalQueueRouteData selectedLogicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); - if (!Objects.equals(selectedLogicalQueueRouteData.getMessageQueue(), new MessageQueue(topic, this.brokerController.getBrokerConfig().getBrokerName(), queueId))) { - log.info("getMaxOffset topic={} queueId={} not latest, redirect: {}", topic, queueId, selectedLogicalQueueRouteData); - response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); - } - } - } finally { - logicalQueuesInfo.readLock().unlock(); - } - } - } + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); + RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; } - long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId, requestHeader.isCommitted()); + long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); responseHeader.setOffset(offset); @@ -662,19 +783,105 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } + private CompletableFuture handleGetMinOffsetForStaticTopic(RpcRequest request, TopicQueueMappingContext mappingContext) { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + if (!mappingContext.isLeader()) { + //this may not + return CompletableFuture.completedFuture(new RpcResponse(new RpcException(ResponseCode.NOT_LEADER_FOR_QUEUE, + String.format("%s-%d is not leader in broker %s, request code %d", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname(), request.getCode())))); + } + GetMinOffsetRequestHeader requestHeader = (GetMinOffsetRequestHeader) request.getHeader(); + LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true); + assert mappingItem != null; + try { + requestHeader.setBname(mappingItem.getBname()); + requestHeader.setLo(false); + requestHeader.setQueueId(mappingItem.getQueueId()); + long physicalOffset; + //run in local + if (mappingItem.getBname().equals(mappingDetail.getBname())) { + physicalOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(mappingDetail.getTopic(), mappingItem.getQueueId()); + } else { + RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_MIN_OFFSET, requestHeader, null); + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + GetMinOffsetResponseHeader offsetResponseHeader = (GetMinOffsetResponseHeader) rpcResponse.getHeader(); + physicalOffset = offsetResponseHeader.getOffset(); + } + long offset = mappingItem.computeStaticQueueOffsetLoosely(physicalOffset); + + final GetMinOffsetResponseHeader responseHeader = new GetMinOffsetResponseHeader(); + responseHeader.setOffset(offset); + return CompletableFuture.completedFuture(new RpcResponse(ResponseCode.SUCCESS, responseHeader, null)); + } catch (Throwable t) { + log.error("rewriteRequestForStaticTopic failed", t); + return CompletableFuture.completedFuture(new RpcResponse(new RpcException(ResponseCode.SYSTEM_ERROR, t.getMessage(), t))); + } + } + + private CompletableFuture handleGetMinOffset(RpcRequest request) { + assert request.getCode() == RequestCode.GET_MIN_OFFSET; + GetMinOffsetRequestHeader requestHeader = (GetMinOffsetRequestHeader) request.getHeader(); + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false); + CompletableFuture rewriteResult = handleGetMinOffsetForStaticTopic(request, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + final GetMinOffsetResponseHeader responseHeader = new GetMinOffsetResponseHeader(); + long offset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + responseHeader.setOffset(offset); + return CompletableFuture.completedFuture(new RpcResponse(ResponseCode.SUCCESS, responseHeader, null)); + } + private RemotingCommand getMinOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { - final RemotingCommand response = RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class); - final GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.readCustomHeader(); final GetMinOffsetRequestHeader requestHeader = (GetMinOffsetRequestHeader) request.decodeCommandCustomHeader(GetMinOffsetRequestHeader.class); + try { + CompletableFuture responseFuture = handleGetMinOffset(new RpcRequest(RequestCode.GET_MIN_OFFSET, requestHeader, null)); + RpcResponse rpcResponse = responseFuture.get(); + return RpcClientUtils.createCommandForRpcResponse(rpcResponse); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } - long offset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + private RemotingCommand rewriteRequestForStaticTopic(GetEarliestMsgStoretimeRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + if (mappingContext.getMappingDetail() == null) { + return null; + } - responseHeader.setOffset(offset); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - return response; + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname())); + } + LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true); + assert mappingItem != null; + try { + requestHeader.setBname(mappingItem.getBname()); + requestHeader.setLo(false); + RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_EARLIEST_MSG_STORETIME, requestHeader, null); + //TO DO check if it is in current broker + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + GetEarliestMsgStoretimeResponseHeader offsetResponseHeader = (GetEarliestMsgStoretimeResponseHeader) rpcResponse.getHeader(); + + final RemotingCommand response = RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class); + final GetEarliestMsgStoretimeResponseHeader responseHeader = (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader(); + responseHeader.setTimestamp(offsetResponseHeader.getTimestamp()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } } private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx, @@ -684,6 +891,12 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements final GetEarliestMsgStoretimeRequestHeader requestHeader = (GetEarliestMsgStoretimeRequestHeader) request.decodeCommandCustomHeader(GetEarliestMsgStoretimeRequestHeader.class); + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false); + RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + long timestamp = this.brokerController.getMessageStore().getEarliestMessageTime(requestHeader.getTopic(), requestHeader.getQueueId()); @@ -806,6 +1019,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } + + private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); @@ -819,8 +1034,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements response.setRemark("topic[" + topic + "] not exist"); return response; } - TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { MessageQueue mq = new MessageQueue(); mq.setTopic(topic); @@ -951,6 +1166,8 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements continue; } + TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic); + { SubscriptionData findSubscriptionData = this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); @@ -978,17 +1195,26 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements requestHeader.getConsumerGroup(), topic, i); - if (consumerOffset < 0) - consumerOffset = 0; + // the consumerOffset cannot be zero for static topic because of the "double read check" strategy + // just remain the logic for dynamic topic + // maybe we should remove it in the future + if (mappingDetail == null) { + if (consumerOffset < 0) + consumerOffset = 0; + } offsetWrapper.setBrokerOffset(brokerOffset); offsetWrapper.setConsumerOffset(consumerOffset); - long timeOffset = consumerOffset - 1; - if (timeOffset >= 0) { - long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); - if (lastTimestamp > 0) { - offsetWrapper.setLastTimestamp(lastTimestamp); + // the consumeOffset is not in this broker for static topic + // and may get the wrong result + if (mappingDetail == null) { + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } } } @@ -1711,11 +1937,16 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(requestHeader.getTopic()); if (topicConfig == null) { log.error("No topic in this broker, client: {} topic: {}", ctx.channel().remoteAddress(), requestHeader.getTopic()); - response.setCode(ResponseCode.SYSTEM_ERROR); + //be care of the response code, should set "not-exist" explictly + response.setCode(ResponseCode.TOPIC_NOT_EXIST); response.setRemark("No topic in this broker. topic: " + requestHeader.getTopic()); return response; } - String content = JSONObject.toJSONString(topicConfig); + TopicQueueMappingDetail topicQueueMappingDetail = null; + if (Boolean.TRUE.equals(requestHeader.getLo())) { + topicQueueMappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(requestHeader.getTopic()); + } + String content = JSONObject.toJSONString(new TopicConfigAndQueueMapping(topicConfig, topicQueueMappingDetail)); try { response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); } catch (UnsupportedEncodingException e) { @@ -1730,588 +1961,4 @@ public class AdminBrokerProcessor extends AsyncNettyRequestProcessor implements return response; } - - private RemotingCommand updateTopicLogicalQueueMapping(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - UpdateTopicLogicalQueueMappingRequestBody requestBody = RemotingSerializable.decode(request.getBody(), UpdateTopicLogicalQueueMappingRequestBody.class); - RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("unknown error"); - if (requestBody == null) { - response.setRemark("decode null"); - return response; - } - String topic = requestBody.getTopic(); - int queueId = requestBody.getQueueId(); - String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); - int logicalQueueIdx = requestBody.getLogicalQueueIdx(); - log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={}", topic, queueId, logicalQueueIdx); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); - if (topicConfig == null) { - log.warn("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={} topic not exist", topic, queueId, logicalQueueIdx); - response.setRemark("topic not exist"); - return response; - } - - LogicalQueuesInfoInBroker logicalQueuesInfo; - LogicalQueueRouteData newQueueRouteData = new LogicalQueueRouteData(); - newQueueRouteData.setBrokerAddr(this.brokerController.getBrokerAddr()); - newQueueRouteData.setMessageQueue(new MessageQueue(topic, brokerName, queueId)); - if (logicalQueueIdx >= 0) { - // add logical queue - newQueueRouteData.setLogicalQueueIndex(logicalQueueIdx); - newQueueRouteData.setLogicalQueueDelta(0L); - newQueueRouteData.setState(MessageQueueRouteState.Normal); - logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - logicalQueuesInfo.writeLock().lock(); - try { - // verify whether this message queue is already set up - for (List queueRouteDataList : logicalQueuesInfo.values()) { - for (Iterator iterator = queueRouteDataList.iterator(); iterator.hasNext(); ) { - LogicalQueueRouteData queueRouteData = iterator.next(); - if (Objects.equals(queueRouteData.getMessageQueue(), newQueueRouteData.getMessageQueue())) { - if (queueRouteData.getLogicalQueueIndex() == logicalQueueIdx) { - log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIndex={} already set up", topic, queueId, logicalQueueIdx); - response.setCode(ResponseCode.SUCCESS); - response.setRemark("already set up"); - return response; - } else { - log.warn("updateTopicLogicalQueueMapping topic={} queueId={} already assigned logicalQueueIdx={}, will reassign as logicalQueueIdx={}", topic, queueRouteData.getMessageQueue(), queueRouteData.getLogicalQueueIndex(), newQueueRouteData.getLogicalQueueIndex()); - iterator.remove(); - break; - } - } - } - } - List queueRouteDataList = logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, ignore -> Lists.newArrayListWithExpectedSize(1)); - int idx = Collections.binarySearch(queueRouteDataList, newQueueRouteData, Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta).thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); - if (idx >= 0) { - log.warn("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIdx={} found same logicalQueueOffset and will replace, exist {}, new {}", topic, queueId, logicalQueueIdx, queueRouteDataList.get(idx), newQueueRouteData); - queueRouteDataList.set(idx, newQueueRouteData); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("duplicate logicalQueueOffset found"); - } else { - idx = -idx - 1; - queueRouteDataList.add(idx, newQueueRouteData); - logicalQueuesInfo.updateQueueRouteDataByQueueId(newQueueRouteData.getQueueId(), newQueueRouteData); - response.setCode(ResponseCode.SUCCESS); - response.setRemark("set up"); - log.info("updateTopicLogicalQueueMapping topic={} queueId={} logicalQueueIdx={} added as #{}", topic, queueId, logicalQueueIdx, idx); - } - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - } else { - // delete logical queue - logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - logicalQueuesInfo.writeLock().lock(); - try { - for (List queueRouteDataList : logicalQueuesInfo.values()) { - queueRouteDataList.removeIf(queueRouteData -> Objects.equals(queueRouteData.getMessageQueue(), newQueueRouteData.getMessageQueue())); - } - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - logicalQueuesInfo.updateQueueRouteDataByQueueId(newQueueRouteData.getQueueId(), null); - this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().remove(new TopicQueueId(topic, queueId)); - response.setCode(ResponseCode.SUCCESS); - response.setRemark("deleted"); - log.info("updateTopicLogicalQueueMapping topic={} queueId={} deleted as logical queue", topic, queueId, logicalQueueIdx); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfig, topicConfigManager.getDataVersion()); - - return response; - } - - private RemotingCommand deleteTopicLogicalQueueMapping(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - DeleteTopicLogicalQueueRequestHeader requestHeader = - (DeleteTopicLogicalQueueRequestHeader) request.decodeCommandCustomHeader(DeleteTopicLogicalQueueRequestHeader.class); - String topic = requestHeader.getTopic(); - log.info("deleteTopicLogicalQueueMapping topic={}", topic); - - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - response.setCode(ResponseCode.SUCCESS); - response.setRemark("already deleted"); - log.info("deleteTopicLogicalQueueMapping topic={} already deleted", topic); - return response; - } - String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); - long size = logicalQueuesInfo.values().stream().flatMap(Collection::stream).filter(v -> Objects.equals(v.getBrokerName(), brokerName)).count(); - if (size > 0) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark(String.format(Locale.ENGLISH, "still %d message queues", size)); - log.info("deleteTopicLogicalQueueMapping topic={} still {} message queues", topic, size); - return response; - } - topicConfigManager.deleteQueueRouteData(topic); - response.setCode(ResponseCode.SUCCESS); - response.setRemark("deleted"); - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - log.info("deleteTopicLogicalQueueMapping topic={} deleted", topic); - return response; - } - - private RemotingCommand queryTopicLogicalQueueMapping(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - QueryTopicLogicalQueueMappingRequestHeader requestHeader = - (QueryTopicLogicalQueueMappingRequestHeader) request.decodeCommandCustomHeader(QueryTopicLogicalQueueMappingRequestHeader.class); - String topic = requestHeader.getTopic(); - log.info("queryTopicLogicalQueueMapping topic={}", topic); - LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); - TreeMap> result = null; - if (logicalQueuesInfo != null) { - result = Maps.newTreeMap(); - logicalQueuesInfo.readLock().lock(); - try { - for (Map.Entry> entry : logicalQueuesInfo.entrySet()) { - Integer k = entry.getKey(); - List v = entry.getValue(); - result.put(k, ImmutableList.copyOf(v)); - } - } finally { - logicalQueuesInfo.readLock().unlock(); - } - } - response.setCode(ResponseCode.SUCCESS); - response.setBody(RemotingSerializable.encode(result)); - return response; - } - - private void sealLogicalQueueRouteData(LogicalQueueRouteData queueRouteData, - MessageStore messageStore) throws TimeoutException, InterruptedException { - queueRouteData.setState(MessageQueueRouteState.ReadOnly); - - String topic = queueRouteData.getTopic(); - int queueId = queueRouteData.getQueueId(); - - // busy wait for all in-fly messages to be finished - TopicQueueId key = new TopicQueueId(topic, queueId); - long startTime = System.currentTimeMillis(); - while (true) { - LongAdder counter = this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().get(key); - if (counter == null || counter.sum() == 0) { - break; - } - if (System.currentTimeMillis() - startTime > 10_000) { - throw new TimeoutException(); - } - Thread.sleep(100); - } - // busy wait for all CQ to be finished - while (messageStore.getMaxOffsetInQueue(topic, queueId, true) != messageStore.getMaxOffsetInQueue(topic, queueId, false)) { - if (System.currentTimeMillis() - startTime > 10_000) { - throw new TimeoutException(); - } - Thread.sleep(100); - } - - long firstMsgQueueOffset = messageStore.getMinOffsetInQueue(topic, queueId); - long lastMsgQueueOffset = messageStore.getMaxOffsetInQueue(topic, queueId, false) - 1; - long firstMsgTimeMillis = 0L; - long lastMsgTimeMillis = 0L; - boolean expired = false; - if (firstMsgQueueOffset == lastMsgQueueOffset) { - // no message at all - expired = true; - } else { - long minPhyOffset = messageStore.getMinPhyOffset(); - long lastMsgCommitLogOffset = lastMsgQueueOffset >= 0 ? messageStore.getCommitLogOffsetInQueue(topic, queueId, lastMsgQueueOffset) : -1; - if (lastMsgCommitLogOffset < minPhyOffset) { - // commitLog already cleaned - expired = true; - } else { - long firstMsgCommitLogOffset = firstMsgQueueOffset >= 0 ? messageStore.getCommitLogOffsetInQueue(topic, queueId, firstMsgQueueOffset) : -1; - MessageExt firstMsg = firstMsgCommitLogOffset >= 0 ? messageStore.lookMessageByOffset(firstMsgCommitLogOffset) : null; - firstMsgTimeMillis = firstMsg != null ? firstMsg.getStoreTimestamp() : 0L; - - MessageExt lastMsg = lastMsgCommitLogOffset >= 0 ? messageStore.lookMessageByOffset(lastMsgCommitLogOffset) : null; - lastMsgTimeMillis = lastMsg != null ? lastMsg.getStoreTimestamp() : 0L; - } - } - - queueRouteData.setOffsetMax(lastMsgQueueOffset + 1); - queueRouteData.setFirstMsgTimeMillis(firstMsgTimeMillis); - queueRouteData.setLastMsgTimeMillis(lastMsgTimeMillis); - - if (expired) { - queueRouteData.setState(MessageQueueRouteState.Expired); - } - } - - private RemotingCommand sealTopicLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { - SealTopicLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), SealTopicLogicalQueueRequestBody.class); - if (requestBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - String topic = requestBody.getTopic(); - int queueId = requestBody.getQueueId(); - int logicalQueueIdx = requestBody.getLogicalQueueIndex(); - log.info("sealTopicLogicalQueue topic={} queueId={}", topic, queueId); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); - if (topicConfig == null) { - log.warn("sealTopicLogicalQueue topic={} queueId={} topic not exist", topic, queueId); - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); - } - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - - LogicalQueueRouteData resultQueueRouteData; - - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - logicalQueuesInfo.readLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); - if (queueRouteDataList == null) { - log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} not exist", topic, queueId, logicalQueueIdx); - response.setRemark("logical queue not exist"); - return response; - } - List foundQueues = queueRouteDataList.stream() - .filter(queueRouteData -> queueId == queueRouteData.getQueueId()).collect(Collectors.toList()); - if (foundQueues.isEmpty()) { - log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} queueId={} not exist", topic, queueId, logicalQueueIdx, queueId); - response.setRemark("message queue not exist"); - return response; - } - Optional firstMainQueueRouteDataOptional = foundQueues.stream().filter(LogicalQueueRouteData::isWritable).findFirst(); - if (!firstMainQueueRouteDataOptional.isPresent()) { - log.info("sealTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} queueId={} already sealed", topic, queueId, logicalQueueIdx, queueId); - response.setRemark("message queue already sealed"); - return response; - } - resultQueueRouteData = firstMainQueueRouteDataOptional.get(); - } finally { - logicalQueuesInfo.readLock().unlock(); - } - try { - sealLogicalQueueRouteData(resultQueueRouteData, brokerController.getMessageStore()); - } catch (InterruptedException e) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "thread interrupted"); - } catch (TimeoutException e) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "seal timeout"); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - response.setBody(RemotingSerializable.encode(resultQueueRouteData)); - return response; - } - - private RemotingCommand reuseTopicLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { - ReuseTopicLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), ReuseTopicLogicalQueueRequestBody.class); - if (requestBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - String topic = requestBody.getTopic(); - int queueId = requestBody.getQueueId(); - String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); - MessageQueue mq = new MessageQueue(topic, brokerName, queueId); - int logicalQueueIdx = requestBody.getLogicalQueueIndex(); - MessageQueueRouteState messageQueueRouteState = requestBody.getMessageQueueRouteState(); - log.info("reuseTopicLogicalQueue topic={} queueId={} logicalQueueIdx={} messageQueueRouteState={}", topic, queueId, logicalQueueIdx, messageQueueRouteState); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "queue route data not found"); - } - if (queueId >= topicConfigManager.selectTopicConfig(topic).getWriteQueueNums()) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "found no queue"); - } - logicalQueuesInfo.writeLock().lock(); - LogicalQueueRouteData queueRouteData = new LogicalQueueRouteData( - logicalQueueIdx, - MessageQueueRouteState.WriteOnly.equals(messageQueueRouteState) ? -1 : 0, - new MessageQueue(topic, brokerName, queueId), - messageQueueRouteState, - this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId, false), - -1, - -1, - -1, - this.brokerController.getBrokerAddr() - ); - try { - if (logicalQueuesInfo.values().stream().flatMap(Collection::stream).filter(v -> Objects.equals(v.getMessageQueue(), mq)).anyMatch(LogicalQueueRouteData::isWritable)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "queue writable"); - } - logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, ignore -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); - logicalQueuesInfo.updateQueueRouteDataByQueueId(queueId, queueRouteData); - this.brokerController.getSendMessageProcessor().getInFlyWritingCounterMap().remove(new TopicQueueId(topic, queueId)); - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setBody(RemotingSerializable.encode(queueRouteData)); - return response; - } - - private RemotingCommand createMessageQueueForLogicalQueue(ChannelHandlerContext ctx, RemotingCommand request) { - CreateMessageQueueForLogicalQueueRequestBody requestBody = RemotingSerializable.decode(request.getBody(), CreateMessageQueueForLogicalQueueRequestBody.class); - if (requestBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - String topic = requestBody.getTopic(); - int logicalQueueIdx = requestBody.getLogicalQueueIndex(); - MessageQueueRouteState messageQueueStatus = requestBody.getMessageQueueStatus(); - log.info("createMessageQueueForLogicalQueue topic={} logicalQueueIdx={}", topic, logicalQueueIdx); - - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo; - LogicalQueueRouteData queueRouteData; - TopicConfig topicConfig; - while (true) { - topicConfig = topicConfigManager.selectTopicConfig(topic); - if (topicConfig == null || topicConfig.getWriteQueueNums() == 0) { - // create topic if not exist - topicConfig = ConcurrentHashMapUtil.computeIfAbsent(topicConfigManager.getTopicConfigTable(), topic, s -> new TopicConfig(topic, 0, 0, this.brokerController.getBrokerConfig().getBrokerPermission())); - logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - } else { - logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - } - if (topicConfig.getWriteQueueNums() > 0 && logicalQueuesInfo == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); - } - TopicConfig newTopicConfig = new TopicConfig(topicConfig); - newTopicConfig.setWriteQueueNums(newTopicConfig.getWriteQueueNums() + 1); - newTopicConfig.setReadQueueNums(newTopicConfig.getReadQueueNums() + 1); - - int queueId = newTopicConfig.getWriteQueueNums() - 1; - queueRouteData = new LogicalQueueRouteData(); - queueRouteData.setLogicalQueueIndex(logicalQueueIdx); - queueRouteData.setBrokerAddr(this.brokerController.getBrokerAddr()); - queueRouteData.setMessageQueue(new MessageQueue(topic, this.brokerController.getBrokerConfig().getBrokerName(), queueId)); - if (messageQueueStatus != null) { - queueRouteData.setState(messageQueueStatus); - switch (messageQueueStatus) { - case WriteOnly: - case Expired: - queueRouteData.setLogicalQueueDelta(-1L); - break; - default: - queueRouteData.setLogicalQueueDelta(0L); - } - } - - logicalQueuesInfo.writeLock().lock(); - try { - List l = logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, i -> Lists.newArrayListWithExpectedSize(1)); - if (MessageQueueRouteState.WriteOnly.equals(messageQueueStatus) && l.stream().anyMatch(LogicalQueueRouteData::isWriteOnly)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "multiple WriteOnly queue"); - } else if (MessageQueueRouteState.Normal.equals(messageQueueStatus) && l.stream().anyMatch(LogicalQueueRouteData::isWritable)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "multiple writable queue"); - } - if (topicConfigManager.replaceTopicConfig(topic, topicConfig, newTopicConfig)) { - l.add(queueRouteData); - logicalQueuesInfo.updateQueueRouteDataByQueueId(queueId, queueRouteData); - break; - } - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfig, topicConfigManager.getDataVersion()); - - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setBody(RemotingSerializable.encode(queueRouteData)); - return response; - } - - private RemotingCommand migrateTopicLogicalQueuePrepare(ChannelHandlerContext ctx, RemotingCommand request) { - MigrateLogicalQueueBody reqRespBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); - if (reqRespBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - LogicalQueueRouteData fromQueueRouteData = reqRespBody.getFromQueueRouteData(); - LogicalQueueRouteData toQueueRouteData = reqRespBody.getToQueueRouteData(); - log.info("migrateTopicLogicalQueuePrepare fromQueueRouteData={} toQueueRouteData={}", fromQueueRouteData, toQueueRouteData); - final MessageQueue fromMessageQueue = fromQueueRouteData.getMessageQueue(); - final int fromQueueId = fromQueueRouteData.getQueueId(); - final long fromOffsetDelta = fromQueueRouteData.getOffsetDelta(); - int logicalQueueIndex = fromQueueRouteData.getLogicalQueueIndex(); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - String topic = fromQueueRouteData.getTopic(); - if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); - } - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); - } - logicalQueuesInfo.writeLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIndex); - if (queueRouteDataList == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "logical queue %d not exist", logicalQueueIndex)); - } - fromQueueRouteData = null; - for (LogicalQueueRouteData v : queueRouteDataList) { - if (v.isSameTo(fromMessageQueue, fromOffsetDelta)) { - fromQueueRouteData = v; - break; - } - } - if (fromQueueRouteData == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d not exist", fromQueueId)); - } - if (!MessageQueueRouteState.Normal.equals(fromQueueRouteData.getState())) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d not normal state", fromQueueId)); - } - reqRespBody.setFromQueueRouteData(fromQueueRouteData); - if (fromQueueRouteData.isWritable()) { - sealLogicalQueueRouteData(fromQueueRouteData, brokerController.getMessageStore()); - } - toQueueRouteData.setLogicalQueueDelta(fromQueueRouteData.getLogicalQueueDelta() + fromQueueRouteData.getMessagesCount()); - queueRouteDataList.add(toQueueRouteData); - } catch (InterruptedException e) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "thread interrupted"); - } catch (TimeoutException e) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "seal timeout"); - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setBody(RemotingSerializable.encode(reqRespBody)); - return response; - } - - private RemotingCommand migrateTopicLogicalQueueCommit(ChannelHandlerContext ctx, RemotingCommand request) { - MigrateLogicalQueueBody reqRespBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); - if (reqRespBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - LogicalQueueRouteData fromQueueRouteData = reqRespBody.getFromQueueRouteData(); - LogicalQueueRouteData toQueueRouteData = reqRespBody.getToQueueRouteData(); - log.info("migrateTopicLogicalQueueCommit toQueueRouteData={}", toQueueRouteData); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - String topic = toQueueRouteData.getTopic(); - int toQueueId = toQueueRouteData.getQueueId(); - if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); - } - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); - } - LogicalQueueRouteData queueRouteData; - logicalQueuesInfo.writeLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(toQueueRouteData.getLogicalQueueIndex()); - if (queueRouteDataList == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "logical queue %d not exist", toQueueRouteData.getLogicalQueueIndex())); - } - queueRouteDataList.stream().filter(fromQueueRouteData::isSameTo).forEach(d -> { - d.copyFrom(fromQueueRouteData); - }); - queueRouteData = queueRouteDataList.stream().filter(toQueueRouteData::isSameTo).findFirst().orElse(null); - if (queueRouteData == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, String.format(Locale.ENGLISH, "message queue %d-%d not exist", toQueueId, toQueueRouteData.getOffsetDelta())); - } - queueRouteData.setLogicalQueueDelta(toQueueRouteData.getLogicalQueueDelta()); - queueRouteData.setState(MessageQueueRouteState.Normal); - toQueueRouteData.setState(MessageQueueRouteState.Normal); - if (toQueueRouteData.getBrokerAddr() != null) { - queueRouteData.setBrokerAddr(toQueueRouteData.getBrokerAddr()); - } - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - response.setBody(RemotingSerializable.encode(reqRespBody)); - return response; - } - - private RemotingCommand migrateTopicLogicalQueueNotify(ChannelHandlerContext ctx, RemotingCommand request) { - MigrateLogicalQueueBody requestBody = RemotingSerializable.decode(request.getBody(), MigrateLogicalQueueBody.class); - if (requestBody == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "decode error"); - } - LogicalQueueRouteData fromQueueRouteData = requestBody.getFromQueueRouteData(); - LogicalQueueRouteData toQueueRouteData = requestBody.getToQueueRouteData(); - log.info("migrateTopicLogicalQueueNotify fromQueueRouteData={} toQueueRouteData={}", fromQueueRouteData, toQueueRouteData); - TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager(); - String topic = toQueueRouteData.getTopic(); - if (!topicConfigManager.getTopicConfigTable().containsKey(topic)) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not exist"); - } - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic); - if (logicalQueuesInfo == null) { - return RemotingCommand.createResponseCommand(ResponseCode.SYSTEM_ERROR, "topic not enable logical queue"); - } - List requestQueueRouteDataList = Lists.newArrayList(fromQueueRouteData, toQueueRouteData); - boolean toQueueRouteDataFound = false; - logicalQueuesInfo.writeLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.computeIfAbsent(toQueueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayListWithExpectedSize(1)); - for (LogicalQueueRouteData v : queueRouteDataList) { - for (Iterator iterator = requestQueueRouteDataList.iterator(); iterator.hasNext(); ) { - LogicalQueueRouteData queueRouteData = iterator.next(); - if (queueRouteData.isSameTo(v)) { - v.copyFrom(queueRouteData); - if (queueRouteData.getBrokerAddr() != null) { - v.setBrokerAddr(queueRouteData.getBrokerAddr()); - } - if (!toQueueRouteDataFound && toQueueRouteData.isSameTo(v)) { - toQueueRouteDataFound = true; - } - iterator.remove(); - break; - } - } - if (requestQueueRouteDataList.isEmpty()) { - break; - } - } - if (!queueRouteDataList.isEmpty() && !toQueueRouteDataFound) { - // if this broker has this logical queue before, it should add latest writable route here, so that SendMessage request can be proxied - queueRouteDataList.add(toQueueRouteData); - } - } finally { - logicalQueuesInfo.writeLock().unlock(); - } - - topicConfigManager.getDataVersion().nextVersion(); - topicConfigManager.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(topicConfigManager.selectTopicConfig(topic), topicConfigManager.getDataVersion()); - - final RemotingCommand response = RemotingCommand.createResponseCommand(null); - response.setCode(ResponseCode.SUCCESS); - return response; - } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java index 77317a6ffe..04e705bf93 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java @@ -17,11 +17,9 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.ChannelHandlerContext; -import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; @@ -31,6 +29,12 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHead import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader; +import org.apache.rocketmq.common.rpc.RpcRequest; +import org.apache.rocketmq.common.rpc.RpcResponse; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -38,6 +42,10 @@ import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import java.util.List; + +import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; + public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -109,6 +117,12 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen final UpdateConsumerOffsetRequestHeader requestHeader = (UpdateConsumerOffsetRequestHeader) request .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); + + RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); response.setCode(ResponseCode.SUCCESS); @@ -116,6 +130,77 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen return response; } + + public RemotingCommand rewriteRequestForStaticTopic(QueryConsumerOffsetRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname())); + } + List mappingItemList = mappingContext.getMappingItemList(); + if (mappingItemList.size() == 1 + && mappingItemList.get(0).getLogicOffset() == 0) { + //as physical, just let it go + requestHeader.setQueueId(mappingContext.getLeaderItem().getQueueId()); + return null; + } + //double read check + List itemList = mappingContext.getMappingItemList(); + //by default, it is -1 + long offset = -1; + //double read, first from leader, then from second leader + for (int i = itemList.size() - 1; i >= 0; i--) { + LogicQueueMappingItem mappingItem = itemList.get(i); + if (mappingItem.getBname().equals(mappingDetail.getBname())) { + offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), requestHeader.getTopic(), mappingItem.getQueueId()); + if (offset >= 0) { + break; + } else { + //not found + continue; + } + } else { + //maybe we need to reconstruct an object + requestHeader.setBname(mappingItem.getBname()); + requestHeader.setQueueId(mappingItem.getQueueId()); + requestHeader.setLo(false); + requestHeader.setSetZeroIfNotFound(false); + RpcRequest rpcRequest = new RpcRequest(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader, null); + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + if (rpcResponse.getCode() == ResponseCode.SUCCESS) { + offset = ((QueryConsumerOffsetResponseHeader) rpcResponse.getHeader()).getOffset(); + break; + } else if (rpcResponse.getCode() == ResponseCode.QUERY_NOT_FOUND) { + continue; + } else { + //this should not happen + throw new RuntimeException("Unknown response code " + rpcResponse.getCode()); + } + } + } + final RemotingCommand response = RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class); + final QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.readCustomHeader(); + if (offset >= 0) { + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } else { + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("Not found, maybe this group consumer boot first"); + } + return response; + } catch (Throwable t) { + t.printStackTrace(); + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = @@ -126,6 +211,13 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen (QueryConsumerOffsetRequestHeader) request .decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class); + + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); + RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + long offset = this.brokerController.getConsumerOffsetManager().queryOffset( requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId()); @@ -138,7 +230,10 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - if (minOffset <= 0 + if (requestHeader.getSetZeroIfNotFound() != null && Boolean.FALSE.equals(requestHeader.getSetZeroIfNotFound())) { + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("Not found, do not set to zero, maybe this group boot first"); + } else if (minOffset <= 0 && !this.brokerController.getMessageStore().checkInDiskByConsumeOffset( requestHeader.getTopic(), requestHeader.getQueueId(), 0)) { responseHeader.setOffset(0L); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index e61ef11309..0f1ca23c1a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -16,17 +16,13 @@ */ package org.apache.rocketmq.broker.processor; -import com.google.common.collect.ImmutableList; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.FileRegion; -import java.nio.ByteBuffer; -import java.util.List; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ConsumerGroupInfo; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter; @@ -43,17 +39,22 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent; +import org.apache.rocketmq.common.rpc.RpcClientUtils; +import org.apache.rocketmq.common.rpc.RpcRequest; +import org.apache.rocketmq.common.rpc.RpcResponse; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.PullSysFlag; @@ -67,7 +68,6 @@ import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.MessageExtBrokerInner; import org.apache.rocketmq.store.MessageFilter; @@ -75,6 +75,11 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import java.nio.ByteBuffer; +import java.util.List; + +import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; + public class PullMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; @@ -95,6 +100,200 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements return false; } + + + private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + String topic = mappingContext.getTopic(); + Integer globalId = mappingContext.getGlobalId(); + // if the leader? consider the order consumer, which will lock the mq + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d cannot find mapping item in request process of current broker %s", topic, globalId, mappingDetail.getBname())); + } + Long globalOffset = requestHeader.getQueueOffset(); + + LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true); + mappingContext.setCurrentItem(mappingItem); + + if (globalOffset < mappingItem.getLogicOffset()) { + //handleOffsetMoved + //If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently + //Otherwise, we could just transfer it to the physical process + } + //below are physical info + String bname = mappingItem.getBname(); + Integer phyQueueId = mappingItem.getQueueId(); + Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset); + requestHeader.setQueueId(phyQueueId); + requestHeader.setQueueOffset(phyQueueOffset); + if (mappingItem.checkIfEndOffsetDecided() + && requestHeader.getMaxMsgNums() != null) { + requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums())); + } + + if (mappingDetail.getBname().equals(bname)) { + //just let it go, do the local pull process + return null; + } + + int sysFlag = requestHeader.getSysFlag(); + requestHeader.setLo(false); + requestHeader.setBname(bname); + sysFlag = PullSysFlag.clearSuspendFlag(sysFlag); + sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag); + requestHeader.setSysFlag(sysFlag); + RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null); + RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + + PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader(); + { + RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode()); + if (rewriteResult != null) { + return rewriteResult; + } + } + return RpcClientUtils.createCommandForRpcResponse(rpcResponse); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + + private RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader, PullMessageResponseHeader responseHeader, + TopicQueueMappingContext mappingContext, final int code) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem(); + + LogicQueueMappingItem currentItem = mappingContext.getCurrentItem(); + + LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true); + + assert currentItem.getLogicOffset() >= 0; + + long requestOffset = requestHeader.getQueueOffset(); + long nextBeginOffset = responseHeader.getNextBeginOffset(); + long minOffset = responseHeader.getMinOffset(); + long maxOffset = responseHeader.getMaxOffset(); + int responseCode = code; + + //consider the following situations + // 1. read from slave, currently not supported + // 2. the middle queue is truncated because of deleting commitlog + if (code != ResponseCode.SUCCESS) { + //note the currentItem maybe both the leader and the earliest + boolean isRevised = false; + if (leaderItem.getGen() == currentItem.getGen()) { + //read the leader + if (requestOffset > maxOffset) { + //actually, we need do nothing, but keep the code structure here + if (code == ResponseCode.PULL_OFFSET_MOVED) { + responseCode = ResponseCode.PULL_OFFSET_MOVED; + nextBeginOffset = maxOffset; + } else { + //maybe current broker is the slave + responseCode = code; + } + } else if (requestOffset < minOffset) { + nextBeginOffset = minOffset; + responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + } else { + responseCode = code; + } + } + //note the currentItem maybe both the leader and the earliest + if (earlistItem.getGen() == currentItem.getGen()) { + //read the earliest one + if (requestOffset < minOffset) { + if (code == ResponseCode.PULL_OFFSET_MOVED) { + responseCode = ResponseCode.PULL_OFFSET_MOVED; + nextBeginOffset = minOffset; + } else { + //maybe read from slave, but we still set it to moved + responseCode = ResponseCode.PULL_OFFSET_MOVED; + nextBeginOffset = minOffset; + } + } else if (requestOffset >= maxOffset) { + //just move to another item + LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true); + if (nextItem != null) { + isRevised = true; + currentItem = nextItem; + nextBeginOffset = currentItem.getStartOffset(); + minOffset = currentItem.getStartOffset(); + maxOffset = minOffset; + responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + } else { + //maybe the next one's logic offset is -1 + responseCode = ResponseCode.PULL_NOT_FOUND; + } + } else { + //let it go + responseCode = code; + } + } + + //read from the middle item, ignore the PULL_OFFSET_MOVED + if (!isRevised + && leaderItem.getGen() != currentItem.getGen() + && earlistItem.getGen() != currentItem.getGen()) { + if (requestOffset < minOffset) { + nextBeginOffset = minOffset; + responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + } else if (requestOffset >= maxOffset) { + //just move to another item + LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true); + if (nextItem != null) { + currentItem = nextItem; + nextBeginOffset = currentItem.getStartOffset(); + minOffset = currentItem.getStartOffset(); + maxOffset = minOffset; + responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY; + } else { + //maybe the next one's logic offset is -1 + responseCode = ResponseCode.PULL_NOT_FOUND; + } + } else { + responseCode = code; + } + } + } + + //handle nextBeginOffset + //the next begin offset should no more than the end offset + if (currentItem.checkIfEndOffsetDecided() + && nextBeginOffset >= currentItem.getEndOffset()) { + nextBeginOffset = currentItem.getEndOffset(); + } + responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset)); + //handle min offset + responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset))); + //handle max offset + responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset), + TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId()))); + //set the offsetDelta + responseHeader.setOffsetDelta(currentItem.computeOffsetDelta()); + + if (code != ResponseCode.SUCCESS) { + return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader); + } else { + return null; + } + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend) throws RemotingCommandException { RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); @@ -132,25 +331,33 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0; - String topic = requestHeader.getTopic(); - TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); if (null == topicConfig) { - log.error("the topic {} not exist, consumer: {}", topic, RemotingHelper.parseChannelRemoteAddr(channel)); + log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel)); response.setCode(ResponseCode.TOPIC_NOT_EXIST); - response.setRemark(String.format("topic[%s] not exist, apply first please! %s", topic, FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); + response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL))); return response; } if (!PermName.isReadable(topicConfig.getPerm())) { response.setCode(ResponseCode.NO_PERMISSION); - response.setRemark("the topic[" + topic + "] pulling message is forbidden"); + response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden"); return response; } - int queueId = requestHeader.getQueueId(); - if (queueId < 0 || queueId >= topicConfig.getReadQueueNums()) { + + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false); + + { + RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + } + + if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) { String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]", - queueId, topic, topicConfig.getReadQueueNums(), channel.remoteAddress()); + requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress()); log.warn(errorInfo); response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark(errorInfo); @@ -162,11 +369,11 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements if (hasSubscriptionFlag) { try { subscriptionData = FilterAPI.build( - topic, requestHeader.getSubscription(), requestHeader.getExpressionType() + requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType() ); if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) { consumerFilterData = ConsumerFilterManager.build( - topic, requestHeader.getConsumerGroup(), requestHeader.getSubscription(), + requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(), requestHeader.getExpressionType(), requestHeader.getSubVersion() ); assert consumerFilterData != null; @@ -195,9 +402,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements return response; } - subscriptionData = consumerGroupInfo.findSubscriptionData(topic); + subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic()); if (null == subscriptionData) { - log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), topic); + log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic()); response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); return response; @@ -211,7 +418,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements return response; } if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) { - consumerFilterData = this.brokerController.getConsumerFilterManager().get(topic, + consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(), requestHeader.getConsumerGroup()); if (consumerFilterData == null) { response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST); @@ -220,7 +427,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements } if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) { log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}", - requestHeader.getConsumerGroup(), topic, consumerFilterData.getClientVersion(), requestHeader.getSubVersion()); + requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion()); response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST); response.setRemark("the consumer's consumer filter data not latest"); return response; @@ -235,6 +442,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements return response; } + MessageFilter messageFilter; if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) { messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData, @@ -244,72 +452,12 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements this.brokerController.getConsumerFilterManager()); } - long offset = requestHeader.getQueueOffset(); - int maxMsgNums = requestHeader.getMaxMsgNums(); - - LogicalQueuesInfoInBroker logicalQueuesInfo = this.brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic); - LogicalQueueRouteData queueRouteData = null; - if (logicalQueuesInfo != null) { - int responseErrorCode = ResponseCode.SUCCESS; - queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, offset); - if (queueRouteData != null) { - if (queueRouteData.isWriteOnly()) { - responseErrorCode = ResponseCode.PULL_NOT_FOUND; - response.setRemark("logical queue write only"); - } else if (queueRouteData.isExpired()) { - responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; - response.setRemark("logical queue expired"); - prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); - } else if (MessageQueueRouteState.ReadOnly.equals(queueRouteData.getState()) && queueRouteData.getOffsetMax() >= 0) { - if (offset >= queueRouteData.getOffsetMax()) { - responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; - response.setRemark("queue offset exceed offsetMax"); - prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); - } else if (offset + maxMsgNums > queueRouteData.getOffsetMax()) { - if ((queueRouteData.getOffsetMax() - 1 <= this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId)) && - (this.brokerController.getMessageStore().getCommitLogOffsetInQueue(topic, queueId, queueRouteData.getOffsetMax() - 1) < this.brokerController.getMessageStore().getMinPhyOffset())) { - responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; - response.setRemark("queue offset removed"); - prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); - } else { - maxMsgNums = (int) (queueRouteData.getOffsetMax() - offset); - if (maxMsgNums <= 0) { - responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; - response.setRemark("queue offset out of range"); - prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); - } - } - } - } - } else { - responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY; - response.setRemark("no suitable queue"); - response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); - // instruct client to refresh all - response.setBody(null); - queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, 0L); - } - if (responseErrorCode != ResponseCode.SUCCESS) { - response.setCode(responseErrorCode); - responseHeader.setMinOffset(offset); - responseHeader.setMaxOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset); - responseHeader.setNextBeginOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset); - responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); - return response; - } - } - final GetMessageResult getMessageResult = - this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic, - queueId, offset, maxMsgNums, messageFilter); + this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(), + requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter); if (getMessageResult != null) { response.setRemark(getMessageResult.getStatus().name()); - long nextBeginOffset = getMessageResult.getNextBeginOffset(); - if (queueRouteData != null && queueRouteData.getOffsetMax() >= 0 && nextBeginOffset > queueRouteData.getOffsetMax()) { - // prevent from pulling messages from next logical queue route data - nextBeginOffset = queueRouteData.getOffsetMax(); - } - responseHeader.setNextBeginOffset(nextBeginOffset); + responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset()); responseHeader.setMinOffset(getMessageResult.getMinOffset()); // this does not need to be modified since it's not an accurate value under logical queue. responseHeader.setMaxOffset(getMessageResult.getMaxOffset()); @@ -360,9 +508,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements // XXX: warn and notify me log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}", requestHeader.getQueueOffset(), - nextBeginOffset, - topic, - queueId, + getMessageResult.getNextBeginOffset(), + requestHeader.getTopic(), + requestHeader.getQueueId(), requestHeader.getConsumerGroup() ); } else { @@ -387,7 +535,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements case OFFSET_TOO_SMALL: response.setCode(ResponseCode.PULL_OFFSET_MOVED); log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}", - requestHeader.getConsumerGroup(), topic, requestHeader.getQueueOffset(), + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(), getMessageResult.getMinOffset(), channel.remoteAddress()); break; default: @@ -395,11 +543,14 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements break; } + + + if (this.hasConsumeMessageHook()) { ConsumeMessageContext context = new ConsumeMessageContext(); context.setConsumerGroup(requestHeader.getConsumerGroup()); - context.setTopic(topic); - context.setQueueId(queueId); + context.setTopic(requestHeader.getTopic()); + context.setQueueId(requestHeader.getQueueId()); String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); @@ -437,6 +588,12 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements this.executeConsumeMessageHookBefore(context); } + //rewrite the response for the + RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode()); + if (rewriteResult != null) { + response = rewriteResult; + } + switch (response.getCode()) { case ResponseCode.SUCCESS: @@ -483,6 +640,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills(); } + String topic = requestHeader.getTopic(); + long offset = requestHeader.getQueueOffset(); + int queueId = requestHeader.getQueueId(); PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills, this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter); this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest); @@ -490,20 +650,6 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements break; } - if (queueRouteData != null) { - logicalQueuesInfo.readLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(queueRouteData.getLogicalQueueIndex()); - MessageQueue latestMessageQueue = queueRouteDataList.get(queueRouteDataList.size() - 1).getMessageQueue(); - if (!latestMessageQueue.getBrokerName().equals(brokerController.getBrokerConfig().getBrokerName()) || latestMessageQueue.getQueueId() != queueId) { - // There are other newer message queue, instruct client to refresh meta-data to access these - prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData); - } - } finally { - logicalQueuesInfo.readLock().unlock(); - } - } - case ResponseCode.PULL_RETRY_IMMEDIATELY: break; case ResponseCode.PULL_OFFSET_MOVED: @@ -518,7 +664,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements event.setConsumerGroup(requestHeader.getConsumerGroup()); event.setMessageQueue(mq); event.setOffsetRequest(requestHeader.getQueueOffset()); - event.setOffsetNew(nextBeginOffset); + event.setOffsetNew(getMessageResult.getNextBeginOffset()); this.generateOffsetMovedEvent(event); log.warn( "PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}", @@ -547,20 +693,11 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; if (storeOffsetEnable) { this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel), - requestHeader.getConsumerGroup(), topic, queueId, requestHeader.getCommitOffset()); + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); } return response; } - private void prepareRedirectResponse(RemotingCommand response, LogicalQueuesInfoInBroker logicalQueuesInfo, - LogicalQueueRouteData queueRouteData) { - LogicalQueueRouteData nextReadableLogicalQueueRouteData = logicalQueuesInfo.nextAvailableLogicalRouteData(queueRouteData, LogicalQueueRouteData::isReadable); - if (nextReadableLogicalQueueRouteData != null) { - response.addExtField(MessageConst.PROPERTY_REDIRECT, "1"); - response.setBody(RemotingSerializable.encode(ImmutableList.of(queueRouteData, nextReadableLogicalQueueRouteData))); - } - } - public boolean hasConsumeMessageHook() { return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 91463a4937..17f19cb1fe 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -28,10 +28,13 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; import org.apache.rocketmq.common.MQVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.help.FAQUrl; @@ -60,6 +63,8 @@ import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.stats.BrokerStatsManager; +import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; + public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { private List consumeMessageHookList; @@ -96,16 +101,54 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement if (requestHeader == null) { return CompletableFuture.completedFuture(null); } + TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true); + RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext); + if (rewriteResult != null) { + return CompletableFuture.completedFuture(rewriteResult); + } mqtraceContext = buildMsgContext(ctx, requestHeader); this.executeSendMessageHookBefore(ctx, request, mqtraceContext); if (requestHeader.isBatch()) { - return this.asyncSendBatchMessage(ctx, request, mqtraceContext, requestHeader); + return this.asyncSendBatchMessage(ctx, request, mqtraceContext, requestHeader, mappingContext); } else { - return this.asyncSendMessage(ctx, request, mqtraceContext, requestHeader); + return this.asyncSendMessage(ctx, request, mqtraceContext, requestHeader, mappingContext); } } } + + /** + * If the response is not null, it meets some errors + * @return + */ + + + private RemotingCommand rewriteResponseForStaticTopic(SendMessageResponseHeader responseHeader, TopicQueueMappingContext mappingContext) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + + LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem(); + if (mappingItem == null) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname())); + } + //no need to care the broker name + long staticLogicOffset = mappingItem.computeStaticQueueOffsetLoosely(responseHeader.getQueueOffset()); + if (staticLogicOffset < 0) { + //if the logic offset is -1, just let it go + //maybe we need a dynamic config + //return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d convert offset error in current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname())); + } + responseHeader.setQueueId(mappingContext.getGlobalId()); + responseHeader.setQueueOffset(staticLogicOffset); + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + return null; + } + @Override public boolean rejectRequest() { return this.brokerController.getMessageStore().isOSPageCacheBusy() || @@ -171,6 +214,14 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement return CompletableFuture.completedFuture(response); } + if (requestHeader.getOriginTopic() != null + && !msgExt.getTopic().equals(requestHeader.getOriginTopic())) { + //here just do some fence in case of some unexpected offset is income + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("look message by offset failed to check the topic name" + requestHeader.getOffset()); + return CompletableFuture.completedFuture(response); + } + final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); if (null == retryTopic) { MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic()); @@ -264,7 +315,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement private CompletableFuture asyncSendMessage(ChannelHandlerContext ctx, RemotingCommand request, SendMessageContext mqtraceContext, - SendMessageRequestHeader requestHeader) { + SendMessageRequestHeader requestHeader, + TopicQueueMappingContext mappingContext) { final RemotingCommand response = preSend(ctx, request, requestHeader); final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); @@ -310,12 +362,6 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); } - LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(msgInner.getTopic(), msgInner.getQueueId(), response); - CompletableFuture future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response); - if (future != null) { - return future; - } - CompletableFuture putMessageResult = null; String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); if (transFlag != null && Boolean.parseBoolean(transFlag)) { @@ -324,15 +370,13 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement response.setRemark( "the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + "] sending transaction message is forbidden"); - logicalQueueContext.hookAfterPut(null); return CompletableFuture.completedFuture(response); } putMessageResult = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner); } else { putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner); } - logicalQueueContext.hookAfterPut(putMessageResult); - return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt); + return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt, requestHeader, mappingContext); } private CompletableFuture handlePutMessageResultFuture(CompletableFuture putMessageResult, @@ -342,9 +386,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, - int queueIdInt) { + int queueIdInt, + SendMessageRequestHeader requestHeader, + TopicQueueMappingContext mappingContext) { return putMessageResult.thenApply((r) -> - handlePutMessageResult(r, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt) + handlePutMessageResult(r, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, requestHeader, mappingContext) ); } @@ -396,7 +442,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement private RemotingCommand sendMessage(final ChannelHandlerContext ctx, final RemotingCommand request, final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + final SendMessageRequestHeader requestHeader, + final TopicQueueMappingContext mappingContext) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); @@ -465,14 +512,15 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); } - return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); + return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, requestHeader, mappingContext); } private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response, RemotingCommand request, MessageExt msg, SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, - int queueIdInt) { + int queueIdInt, SendMessageRequestHeader requestHeader, + TopicQueueMappingContext mappingContext) { if (putMessageResult == null) { response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("store putMessage return null"); @@ -549,6 +597,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement responseHeader.setQueueId(queueIdInt); responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext); + if (rewriteResult != null) { + return rewriteResult; + } + doResponse(ctx, request, response); if (hasSendMessageHook()) { @@ -582,7 +635,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement private CompletableFuture asyncSendBatchMessage(ChannelHandlerContext ctx, RemotingCommand request, SendMessageContext mqtraceContext, - SendMessageRequestHeader requestHeader) { + SendMessageRequestHeader requestHeader, + TopicQueueMappingContext mappingContext) { final RemotingCommand response = preSend(ctx, request, requestHeader); final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); @@ -623,17 +677,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName); - LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(messageExtBatch.getTopic(), messageExtBatch.getQueueId(), response); - CompletableFuture future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response); - if (future != null) { - return future; - } - CompletableFuture putMessageResult = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch); - - logicalQueueContext.hookAfterPut(putMessageResult); - - return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt); + return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt, requestHeader, mappingContext); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java index 7b5e5645a0..9bb09bd877 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java @@ -16,18 +16,19 @@ */ package org.apache.rocketmq.broker.slave; -import java.io.IOException; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import java.io.IOException; + public class SlaveSynchronize { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); private final BrokerController brokerController; @@ -56,7 +57,7 @@ public class SlaveSynchronize { String masterAddrBak = this.masterAddr; if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) { try { - TopicConfigSerializeWrapper topicWrapper = + TopicConfigAndMappingSerializeWrapper topicWrapper = this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak); if (!this.brokerController.getTopicConfigManager().getDataVersion() .equals(topicWrapper.getDataVersion())) { @@ -67,9 +68,17 @@ public class SlaveSynchronize { this.brokerController.getTopicConfigManager().getTopicConfigTable() .putAll(topicWrapper.getTopicConfigTable()); this.brokerController.getTopicConfigManager().persist(); - - log.info("Update slave topic config from master, {}", masterAddrBak); } + if (topicWrapper.getTopicQueueMappingDetailMap() != null + && !topicWrapper.getMappingDataVersion().equals(this.brokerController.getTopicQueueMappingManager().getDataVersion())) { + this.brokerController.getTopicQueueMappingManager().getDataVersion() + .assignNewOne(topicWrapper.getMappingDataVersion()); + this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable().clear(); + this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable() + .putAll(topicWrapper.getTopicQueueMappingDetailMap()); + this.brokerController.getTopicQueueMappingManager().persist(); + } + log.info("Update slave topic config from master, {}", masterAddrBak); } catch (Exception e) { log.error("SyncTopicConfig Exception, {}", masterAddrBak, e); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index bf690057c6..4af230d453 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -17,20 +17,16 @@ package org.apache.rocketmq.broker.topic; import java.util.Iterator; -import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.BrokerPathConfigHelper; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.common.ConfigManager; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; @@ -39,15 +35,10 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; -import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; -import org.apache.rocketmq.store.CleanFilesHook; -import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.MessageStore; public class TopicConfigManager extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -61,19 +52,8 @@ public class TopicConfigManager extends ConfigManager { private final DataVersion dataVersion = new DataVersion(); private transient BrokerController brokerController; - private final ConcurrentMap logicalQueuesInfoTable = new ConcurrentHashMap<>(); - private final CleanFilesHook logicalQueueCleanHook = new CleanFilesHook() { - @Override public void execute(DefaultMessageStore defaultMessageStore, long deleteCount) { - if (deleteCount == 0) { - return; - } - TopicConfigManager.this.logicalQueueClean(); - } - - @Override public String getName() { - return TopicConfigManager.class.getSimpleName() + ".logicalQueueCleanHook"; - } - }; + public TopicConfigManager() { + } public TopicConfigManager(BrokerController brokerController) { this.brokerController = brokerController; @@ -441,8 +421,6 @@ public class TopicConfigManager extends ConfigManager { public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() { TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); - String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); - topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue(), data -> Objects.equals(data.getBrokerName(), brokerName))))); topicConfigSerializeWrapper.setDataVersion(this.dataVersion); return topicConfigSerializeWrapper; } @@ -474,7 +452,6 @@ public class TopicConfigManager extends ConfigManager { public String encode(final boolean prettyFormat) { TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); - topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue())))); topicConfigSerializeWrapper.setDataVersion(this.dataVersion); return topicConfigSerializeWrapper.toJson(prettyFormat); } @@ -494,92 +471,4 @@ public class TopicConfigManager extends ConfigManager { public ConcurrentMap getTopicConfigTable() { return topicConfigTable; } - - public LogicalQueuesInfoInBroker selectLogicalQueuesInfo(String topicName) { - return this.logicalQueuesInfoTable.get(topicName); - } - - public LogicalQueuesInfoInBroker getOrCreateLogicalQueuesInfo(String topicName) { - return ConcurrentHashMapUtil.computeIfAbsent(this.logicalQueuesInfoTable, topicName, ignored -> new LogicalQueuesInfoInBroker()); - } - - public boolean replaceTopicConfig(String topic, TopicConfig oldTopicConfig, TopicConfig newTopicConfig) { - boolean ok = this.topicConfigTable.replace(topic, oldTopicConfig, newTopicConfig); - if (ok) { - this.dataVersion.nextVersion(); - persist(topic, newTopicConfig); - } - return ok; - } - - public CleanFilesHook getLogicalQueueCleanHook() { - return logicalQueueCleanHook; - } - - void logicalQueueClean() { - String brokerName = this.brokerController.getBrokerConfig().getBrokerName(); - MessageStore messageStore = this.brokerController.getMessageStore(); - for (Entry entry : this.logicalQueuesInfoTable.entrySet()) { - String topic = entry.getKey(); - LogicalQueuesInfoInBroker logicalQueuesInfo = entry.getValue(); - Lock readLock = logicalQueuesInfo.readLock(); - Lock writeLock = logicalQueuesInfo.writeLock(); - boolean changed = false; - readLock.lock(); - try { - for (List list : logicalQueuesInfo.values()) { - while (!list.isEmpty()) { - LogicalQueueRouteData logicalQueueRouteData = list.get(0); - String brokerBelongs; - if (brokerName.equals(logicalQueueRouteData.getBrokerName())) { - if (logicalQueueRouteData.isWritable()) { - break; - } - boolean canRemove = logicalQueueRouteData.isExpired() || logicalQueueRouteData.getMessagesCount() == 0; - if (!canRemove) { - // do not use getMinOffsetInQueue method, since it is using ConsumeQueue data, but not CommitLog, CQ data is not accurate after CommitLog cleaning. - long commitLogOffset = messageStore.getCommitLogOffsetInQueue(topic, logicalQueueRouteData.getQueueId(), logicalQueueRouteData.getOffsetMax() - 1); - canRemove = commitLogOffset == 0 || messageStore.getMinPhyOffset() > commitLogOffset; - } - if (!canRemove) { - break; - } - brokerBelongs = "self"; - } else { - brokerBelongs = "other"; - } - readLock.unlock(); - writeLock.lock(); - try { - list.remove(0); - } finally { - readLock.lock(); - writeLock.unlock(); - } - log.info("logicalQueueClean remove {} broker {}", brokerBelongs, logicalQueueRouteData); - changed = true; - } - } - if (changed) { - logicalQueuesInfo = new LogicalQueuesInfoInBroker(logicalQueuesInfo); - } - } finally { - readLock.unlock(); - } - if (changed) { - this.dataVersion.nextVersion(); - this.persist(topic, logicalQueuesInfo); - this.brokerController.registerIncrementBrokerData(this.selectTopicConfig(topic), this.dataVersion); - log.info("registerIncrementBrokerData because logicalQueueClean: {}", topic); - } - } - } - - public void deleteQueueRouteData(String topic) { - if (this.logicalQueuesInfoTable.remove(topic) != null) { - log.info("delete queueRouteData config OK, topic: {}", topic); - this.dataVersion.nextVersion(); - persist(topic, (LogicalQueuesInfoInBroker) null); - } - } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java new file mode 100644 index 0000000000..0533668cd5 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.topic; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.out.BrokerOuterAPI; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.admin.TopicOffset; +import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.rpc.RpcClient; +import org.apache.rocketmq.common.rpc.RpcRequest; +import org.apache.rocketmq.common.rpc.RpcResponse; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.store.config.MessageStoreConfig; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class TopicQueueMappingCleanService extends ServiceThread { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + + private TopicQueueMappingManager topicQueueMappingManager; + private BrokerOuterAPI brokerOuterAPI; + private RpcClient rpcClient; + private MessageStoreConfig messageStoreConfig; + private BrokerConfig brokerConfig; + + public TopicQueueMappingCleanService(BrokerController brokerController) { + this.topicQueueMappingManager = brokerController.getTopicQueueMappingManager(); + this.rpcClient = brokerController.getBrokerOuterAPI().getRpcClient(); + this.messageStoreConfig = brokerController.getMessageStoreConfig(); + this.brokerConfig = brokerController.getBrokerConfig(); + this.brokerOuterAPI = brokerController.getBrokerOuterAPI(); + } + + @Override + public String getServiceName() { + return TopicQueueMappingCleanService.class.getSimpleName(); + } + + @Override + public void run() { + log.info("Start topic queue mapping clean service thread!"); + while (!this.isStopped()) { + try { + this.waitForRunning(5L * 60 * 1000); + } catch (Throwable ignored) { + } + try { + cleanItemExpired(); + } catch (Throwable t) { + log.error("topic queue mapping cleanItemExpired failed", t); + } + try { + cleanItemListMoreThanSecondGen(); + } catch (Throwable t) { + log.error("topic queue mapping cleanItemListMoreThanSecondGen failed", t); + } + + } + log.info("End topic queue mapping clean service thread!"); + } + + + + public void cleanItemExpired() { + String when = messageStoreConfig.getDeleteWhen(); + if (!UtilAll.isItTimeToDo(when)) { + return; + } + boolean changed = false; + long start = System.currentTimeMillis(); + try { + for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) { + try { + if (isStopped()) { + break; + } + TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic); + if (mappingDetail == null + || mappingDetail.getHostedQueues().isEmpty()) { + continue; + } + if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) { + log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail); + continue; + } + Set brokers = new HashSet<>(); + for (List items: mappingDetail.getHostedQueues().values()) { + if (items.size() <= 1) { + continue; + } + if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) { + continue; + } + LogicQueueMappingItem earlistItem = items.get(0); + brokers.add(earlistItem.getBname()); + } + Map statsTable = new HashMap<>(); + for (String broker: brokers) { + GetTopicStatsInfoRequestHeader header = new GetTopicStatsInfoRequestHeader(); + header.setTopic(topic); + header.setBname(broker); + header.setLo(false); + try { + RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_STATS_INFO, header, null); + RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + statsTable.put(broker, (TopicStatsTable) rpcResponse.getBody()); + } catch (Throwable rt) { + log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt); + } + } + Map> newHostedQueues = new HashMap<>(); + boolean changedForTopic = false; + for (Map.Entry> entry : mappingDetail.getHostedQueues().entrySet()) { + Integer qid = entry.getKey(); + List items = entry.getValue(); + if (items.size() <= 1) { + continue; + } + if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) { + continue; + } + LogicQueueMappingItem earlistItem = items.get(0); + TopicStatsTable topicStats = statsTable.get(earlistItem.getBname()); + if (topicStats == null) { + continue; + } + TopicOffset topicOffset = topicStats.getOffsetTable().get(new MessageQueue(topic, earlistItem.getBname(), earlistItem.getQueueId())); + if (topicOffset == null) { + //this may should not happen + log.error("Get null topicOffset for {} {}",topic, earlistItem); + continue; + } + //ignore the maxOffset < 0, which may in case of some error + if (topicOffset.getMaxOffset() == topicOffset.getMinOffset() + || topicOffset.getMaxOffset() == 0) { + List newItems = new ArrayList<>(items); + boolean result = newItems.remove(earlistItem); + if (result) { + changedForTopic = true; + newHostedQueues.put(qid, newItems); + } + log.info("The logic queue item {} {} is removed {} because of {}", topic, earlistItem, result, topicOffset); + } + } + if (changedForTopic) { + TopicQueueMappingDetail newMappingDetail = new TopicQueueMappingDetail(mappingDetail.getTopic(), mappingDetail.getTotalQueues(), mappingDetail.getBname(), mappingDetail.getEpoch()); + newMappingDetail.getHostedQueues().putAll(mappingDetail.getHostedQueues()); + newMappingDetail.getHostedQueues().putAll(newHostedQueues); + this.topicQueueMappingManager.updateTopicQueueMapping(newMappingDetail, false, true, false); + changed = true; + } + } catch (Throwable tt) { + log.error("Try CleanItemExpired failed for {}", topic, tt); + } finally { + UtilAll.sleep(10); + } + } + } catch (Throwable t) { + log.error("Try cleanItemExpired failed", t); + } finally { + if (changed) { + this.topicQueueMappingManager.getDataVersion().nextVersion(); + this.topicQueueMappingManager.persist(); + log.info("CleanItemExpired changed"); + } + log.info("cleanItemExpired cost {} ms", System.currentTimeMillis() - start); + } + } + + public void cleanItemListMoreThanSecondGen() { + String when = messageStoreConfig.getDeleteWhen(); + if (!UtilAll.isItTimeToDo(when)) { + return; + } + boolean changed = false; + long start = System.currentTimeMillis(); + try { + ClientMetadata clientMetadata = new ClientMetadata(); + for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) { + try { + if (isStopped()) { + break; + } + TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic); + if (mappingDetail == null + || mappingDetail.getHostedQueues().isEmpty()) { + continue; + } + if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) { + log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail); + continue; + } + Map qid2CurrLeaderBroker = new HashMap<>(); + for (Map.Entry> entry : mappingDetail.getHostedQueues().entrySet()) { + Integer qId = entry.getKey(); + List items = entry.getValue(); + if (items.isEmpty()) { + continue; + } + LogicQueueMappingItem leaderItem = items.get(items.size() - 1); + if (!leaderItem.getBname().equals(mappingDetail.getBname())) { + qid2CurrLeaderBroker.put(qId, leaderItem.getBname()); + } + } + if (qid2CurrLeaderBroker.isEmpty()) { + continue; + } + //find the topic route + TopicRouteData topicRouteData = brokerOuterAPI.getTopicRouteInfoFromNameServer(topic, brokerConfig.getForwardTimeout()); + clientMetadata.freshTopicRoute(topic, topicRouteData); + Map qid2RealLeaderBroker = new HashMap<>(); + //fine the real leader + for (Map.Entry entry : qid2CurrLeaderBroker.entrySet()) { + qid2RealLeaderBroker.put(entry.getKey(), clientMetadata.getBrokerNameFromMessageQueue(new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(mappingDetail.getScope()), entry.getKey()))); + } + + //find the mapping detail of real leader + Map mappingDetailMap = new HashMap<>(); + for (Map.Entry entry : qid2RealLeaderBroker.entrySet()) { + if (entry.getValue().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) { + continue; + } + String broker = entry.getValue(); + GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader(); + header.setTopic(topic); + header.setBname(broker); + header.setLo(true); + try { + RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_CONFIG, header, null); + RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get(); + if (rpcResponse.getException() != null) { + throw rpcResponse.getException(); + } + TopicQueueMappingDetail mappingDetailRemote = ((TopicConfigAndQueueMapping) rpcResponse.getBody()).getMappingDetail(); + if (broker.equals(mappingDetailRemote.getBname())) { + mappingDetailMap.put(broker, mappingDetailRemote); + } + } catch (Throwable rt) { + log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt); + } + } + //check all the info + Set ids2delete = new HashSet<>(); + for (Map.Entry entry : qid2CurrLeaderBroker.entrySet()) { + Integer qId = entry.getKey(); + String currLeaderBroker = entry.getValue(); + String realLeaderBroker = qid2RealLeaderBroker.get(qId); + TopicQueueMappingDetail remoteMappingDetail = mappingDetailMap.get(realLeaderBroker); + if (remoteMappingDetail == null + || remoteMappingDetail.getTotalQueues() != mappingDetail.getTotalQueues() + || remoteMappingDetail.getEpoch() != mappingDetail.getEpoch()) { + continue; + } + List items = remoteMappingDetail.getHostedQueues().get(qId); + if (items.isEmpty()) { + continue; + } + LogicQueueMappingItem leaderItem = items.get(items.size() - 1); + if (!realLeaderBroker.equals(leaderItem.getBname())) { + continue; + } + //all the check is ok + if (!realLeaderBroker.equals(currLeaderBroker)) { + ids2delete.add(qId); + } + } + for (Integer qid : ids2delete) { + List items = mappingDetail.getHostedQueues().remove(qid); + changed = true; + if (items != null) { + log.info("Remove the ItemListMoreThanSecondGen topic {} qid {} items {}", topic, qid, items); + } + } + } catch (Throwable tt) { + log.error("Try cleanItemListMoreThanSecondGen failed for topic {}", topic, tt); + } finally { + UtilAll.sleep(10); + } + } + } catch (Throwable t) { + log.error("Try cleanItemListMoreThanSecondGen failed", t); + } finally { + if (changed) { + this.topicQueueMappingManager.getDataVersion().nextVersion(); + this.topicQueueMappingManager.persist(); + } + log.info("Try cleanItemListMoreThanSecondGen cost {} ms", System.currentTimeMillis() - start); + } + } + + + + +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java new file mode 100644 index 0000000000..56fc792709 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.topic; + +import com.alibaba.fastjson.JSON; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.BrokerPathConfigHelper; +import org.apache.rocketmq.common.ConfigManager; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.TopicQueueMappingSerializeWrapper; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; +import org.apache.rocketmq.common.rpc.TopicRequestHeader; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; + +public class TopicQueueMappingManager extends ConfigManager { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final long LOCK_TIMEOUT_MILLIS = 3000; + private transient final Lock lock = new ReentrantLock(); + + //this data version should be equal to the TopicConfigManager + private final DataVersion dataVersion = new DataVersion(); + private transient BrokerController brokerController; + + private final ConcurrentMap topicQueueMappingTable = new ConcurrentHashMap<>(); + + + public TopicQueueMappingManager(BrokerController brokerController) { + this.brokerController = brokerController; + } + + public void updateTopicQueueMapping(TopicQueueMappingDetail newDetail, boolean force, boolean isClean, boolean flush) throws Exception { + boolean locked = false; + boolean updated = false; + TopicQueueMappingDetail oldDetail = null; + try { + + if (lock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { + locked = true; + } else { + return; + } + if (newDetail == null) { + return; + } + assert newDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName()); + + newDetail.getHostedQueues().forEach((queueId, items) -> { + TopicQueueMappingUtils.checkLogicQueueMappingItemOffset(items); + }); + + oldDetail = topicQueueMappingTable.get(newDetail.getTopic()); + if (oldDetail == null) { + topicQueueMappingTable.put(newDetail.getTopic(), newDetail); + updated = true; + return; + } + if (force) { + //bakeup the old items + oldDetail.getHostedQueues().forEach((queueId, items) -> { + newDetail.getHostedQueues().putIfAbsent(queueId, items); + }); + topicQueueMappingTable.put(newDetail.getTopic(), newDetail); + updated = true; + return; + } + //do more check + if (newDetail.getEpoch() < oldDetail.getEpoch()) { + throw new RuntimeException(String.format("Can't accept data with small epoch %d < %d", newDetail.getEpoch(), oldDetail.getEpoch())); + } + if (!newDetail.getScope().equals(oldDetail.getScope())) { + throw new RuntimeException(String.format("Can't accept data with unmatched scope %s != %s", newDetail.getScope(), oldDetail.getScope())); + } + boolean epochEqual = newDetail.getEpoch() == oldDetail.getEpoch(); + for (Integer globalId : oldDetail.getHostedQueues().keySet()) { + List oldItems = oldDetail.getHostedQueues().get(globalId); + List newItems = newDetail.getHostedQueues().get(globalId); + if (newItems == null) { + if (epochEqual) { + throw new RuntimeException("Cannot accept equal epoch with null data"); + } else { + newDetail.getHostedQueues().put(globalId, oldItems); + } + } else { + TopicQueueMappingUtils.makeSureLogicQueueMappingItemImmutable(oldItems, newItems, epochEqual, isClean); + } + } + topicQueueMappingTable.put(newDetail.getTopic(), newDetail); + updated = true; + } finally { + if (locked) { + this.lock.unlock(); + } + if (updated && flush) { + this.dataVersion.nextVersion(); + this.persist(); + log.info("Update topic queue mapping from [{}] to [{}], force {}", oldDetail, newDetail, force); + } + } + + } + + public void delete(final String topic) { + TopicQueueMappingDetail old = this.topicQueueMappingTable.remove(topic); + if (old != null) { + log.info("delete topic queue mapping OK, topic queue mapping: {}", old); + this.dataVersion.nextVersion(); + this.persist(); + } else { + log.warn("delete topic queue mapping failed, topic: {} not exists", topic); + } + } + + public TopicQueueMappingDetail getTopicQueueMapping(String topic) { + return topicQueueMappingTable.get(topic); + } + + @Override + public String encode(boolean pretty) { + TopicQueueMappingSerializeWrapper wrapper = new TopicQueueMappingSerializeWrapper(); + wrapper.setTopicQueueMappingInfoMap(topicQueueMappingTable); + wrapper.setDataVersion(this.dataVersion); + return JSON.toJSONString(wrapper, pretty); + } + + @Override + public String encode() { + return encode(false); + } + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getTopicQueueMappingPath(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + } + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + TopicQueueMappingSerializeWrapper wrapper = TopicQueueMappingSerializeWrapper.fromJson(jsonString, TopicQueueMappingSerializeWrapper.class); + if (wrapper != null) { + this.topicQueueMappingTable.putAll(wrapper.getTopicQueueMappingInfoMap()); + this.dataVersion.assignNewOne(wrapper.getDataVersion()); + } + } + } + + public ConcurrentMap getTopicQueueMappingTable() { + return topicQueueMappingTable; + } + + public DataVersion getDataVersion() { + return dataVersion; + } + + public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader) { + return buildTopicQueueMappingContext(requestHeader, false); + } + + //Do not return a null context + public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader, boolean selectOneWhenMiss) { + //should disable logic queue explicitly, otherwise the old client may cause dirty data to newly created static topic + if (requestHeader.getLo() != null + && Boolean.FALSE.equals(requestHeader.getLo())) { + return new TopicQueueMappingContext(requestHeader.getTopic(), null, null, null, null); + } + String topic = requestHeader.getTopic(); + Integer globalId = null; + if (requestHeader instanceof TopicQueueRequestHeader) { + globalId = ((TopicQueueRequestHeader) requestHeader).getQueueId(); + } + + TopicQueueMappingDetail mappingDetail = getTopicQueueMapping(topic); + if (mappingDetail == null) { + //it is not static topic + return new TopicQueueMappingContext(topic, null, null, null, null); + } + assert mappingDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName()); + + if (globalId == null) { + return new TopicQueueMappingContext(topic, null, mappingDetail, null, null); + } + + //If not find mappingItem, it encounters some errors + if (globalId < 0 && !selectOneWhenMiss) { + return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null); + } + + if (globalId < 0) { + try { + if (!mappingDetail.getHostedQueues().isEmpty()) { + //do not check + globalId = mappingDetail.getHostedQueues().keySet().iterator().next(); + } + } catch (Throwable ignored) { + } + } + if (globalId < 0) { + return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null); + } + + List mappingItemList = TopicQueueMappingDetail.getMappingInfo(mappingDetail, globalId); + LogicQueueMappingItem leaderItem = null; + if (mappingItemList != null + && mappingItemList.size() > 0) { + leaderItem = mappingItemList.get(mappingItemList.size() - 1); + } + return new TopicQueueMappingContext(topic, globalId, mappingDetail, mappingItemList, leaderItem); + } + + + public RemotingCommand rewriteRequestForStaticTopic(TopicQueueRequestHeader requestHeader, TopicQueueMappingContext mappingContext) { + try { + if (mappingContext.getMappingDetail() == null) { + return null; + } + TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail(); + if (!mappingContext.isLeader()) { + return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname())); + } + LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem(); + requestHeader.setQueueId(mappingItem.getQueueId()); + return null; + } catch (Throwable t) { + return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } + } + +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 339ed110c8..daf771ab41 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -84,7 +84,7 @@ public class BrokerOuterAPITest { private BrokerOuterAPI brokerOuterAPI; public void init() throws Exception { - brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), null); + brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), brokerController); Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient"); field.setAccessible(true); field.set(brokerOuterAPI, nettyRemotingClient); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java index c8da08d28d..ce370a39c1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java @@ -20,6 +20,7 @@ package org.apache.rocketmq.broker; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Properties; +import org.apache.rocketmq.common.MixAll; import org.junit.Assert; import org.junit.Test; @@ -34,8 +35,21 @@ public class BrokerStartupTest { Class clazz = BrokerStartup.class; Method method = clazz.getDeclaredMethod("properties2SystemEnv", Properties.class); method.setAccessible(true); - System.setProperty("rocketmq.namesrv.domain", "value"); - method.invoke(null, properties); - Assert.assertEquals("value", System.getProperty("rocketmq.namesrv.domain")); + { + properties.put("rmqAddressServerDomain", "value1"); + properties.put("rmqAddressServerSubGroup", "value2"); + method.invoke(null, properties); + Assert.assertEquals("value1", System.getProperty("rocketmq.namesrv.domain")); + Assert.assertEquals("value2", System.getProperty("rocketmq.namesrv.domain.subgroup")); + } + { + properties.put("rmqAddressServerDomain", MixAll.WS_DOMAIN_NAME); + properties.put("rmqAddressServerSubGroup", MixAll.WS_DOMAIN_SUBGROUP); + method.invoke(null, properties); + Assert.assertEquals(MixAll.WS_DOMAIN_NAME, System.getProperty("rocketmq.namesrv.domain")); + Assert.assertEquals(MixAll.WS_DOMAIN_SUBGROUP, System.getProperty("rocketmq.namesrv.domain.subgroup")); + } + + } } \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java index d6cc8f9bd8..2141f2c999 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java @@ -16,22 +16,10 @@ */ package org.apache.rocketmq.broker.processor; -import com.alibaba.fastjson.JSON; -import com.alibaba.fastjson.TypeReference; import com.google.common.collect.Sets; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.LongAdder; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.topic.TopicConfigManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; @@ -42,28 +30,17 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; -import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; -import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; -import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; -import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader; -import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.store.AppendMessageResult; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.MappedFile; @@ -73,7 +50,6 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -81,13 +57,16 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.LongAdder; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -122,8 +101,7 @@ public class AdminBrokerProcessorTest { public void init() throws Exception { brokerController.setMessageStore(messageStore); - doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor(); - when(sendMessageProcessor.getInFlyWritingCounterMap()).thenReturn(inFlyWritingCouterMap); + //doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor(); adminBrokerProcessor = new AdminBrokerProcessor(brokerController); @@ -225,271 +203,13 @@ public class AdminBrokerProcessorTest { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader); request.makeCustomHeaderToNet(); RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); + assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST); assertThat(response.getRemark()).contains("No topic in this broker."); } } - @Test - public void testUpdateTopicLogicalQueueMapping() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null); - UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody(); - requestBody.setTopic(topic); - requestBody.setQueueId(queueId); - requestBody.setLogicalQueueIdx(logicalQueueIndex); - request.setBody(requestBody.encode()); - RemotingCommand response; - response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())); - // delete - requestBody.setLogicalQueueIdx(-1); - request.setBody(requestBody.encode()); - response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex)).isEmpty(); - verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId)); - } - - @Test - public void testDeleteTopicLogicalQueueMapping() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); - - DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader(); - requestHeader.setTopic(topic); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); - request.makeCustomHeaderToNet(); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(response.getRemark()).isEqualTo("still 1 message queues"); - - logicalQueuesInfo.remove(logicalQueueIndex); - response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull(); - } - - @Test - public void testQueryTopicLogicalQueueMapping() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); - - QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader(); - requestHeader.setTopic(topic); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); - request.makeCustomHeaderToNet(); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - Map> m = JSON.parseObject(response.getBody(), new TypeReference>>() { - }.getType()); - assertThat(m.get(logicalQueueIndex)).isEqualTo(logicalQueuesInfo.get(logicalQueueIndex)); - } - - @Test - public void testSealTopicLogicalQueue() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); - - when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L); - when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(0L); - when(messageStore.getMinPhyOffset()).thenReturn(1000L); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(0L))).thenReturn(2000L); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L); - MessageExt firstMsg = mock(MessageExt.class); - when(firstMsg.getStoreTimestamp()).thenReturn(200L); - when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg); - MessageExt lastMsg = mock(MessageExt.class); - when(lastMsg.getStoreTimestamp()).thenReturn(300L); - when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg); - - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null); - SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody(); - requestBody.setTopic(topic); - requestBody.setQueueId(queueId); - requestBody.setLogicalQueueIndex(logicalQueueIndex); - request.setBody(requestBody.encode()); - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.ReadOnly, 0, 100, 200, 300, brokerController.getBrokerAddr()); - assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); - assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); - - // expired - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); - when(messageStore.getMinPhyOffset()).thenReturn(10000L); - response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr()); - assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); - assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); - - // expired and empty - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()))); - when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(100L); - response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr()); - assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); - assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); - } - - @Test - public void testReuseTopicLogicalQueue() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); - LogicalQueueRouteData logicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Expired, 100L, 200L, 300L, 400L, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(logicalQueueRouteData)); - LogicalQueueRouteData wantData0 = new LogicalQueueRouteData(logicalQueueRouteData); - - when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(600L); - - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null); - ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody(); - requestBody.setTopic(topic); - requestBody.setQueueId(queueId); - requestBody.setLogicalQueueIndex(logicalQueueIndex); - requestBody.setMessageQueueRouteState(MessageQueueRouteState.WriteOnly); - request.setBody(requestBody.encode()); - - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - LogicalQueueRouteData wantData1 = new LogicalQueueRouteData(logicalQueueIndex, -1L, mq, MessageQueueRouteState.WriteOnly, 600L, -1, -1, -1, brokerController.getBrokerAddr()); - assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantData1); - assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Arrays.asList(wantData0, wantData1)); - verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId)); - } - - @Test - public void testCreateMessageQueueForLogicalQueue() throws Exception { - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic); - topicConfig.setWriteQueueNums(0); - topicConfig.setReadQueueNums(0); - int queueId = 0; - assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull(); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null); - CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody(); - requestBody.setTopic(topic); - requestBody.setLogicalQueueIndex(logicalQueueIndex); - requestBody.setMessageQueueStatus(MessageQueueRouteState.WriteOnly); - request.setBody(requestBody.encode()); - - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); - LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, -1L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.WriteOnly, 0L, -1, -1, -1, brokerController.getBrokerAddr()); - assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData); - assertThat(topicConfigManager.selectLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData); - } - - @Test - public void testMigrateTopicLogicalQueuePrepare() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); - LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 10L, -1L, -1L, -1L, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData))); - - when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L); - when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(10L); - when(messageStore.getMinPhyOffset()).thenReturn(1000L); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(10L))).thenReturn(2000L); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L); - MessageExt firstMsg = mock(MessageExt.class); - when(firstMsg.getStoreTimestamp()).thenReturn(200L); - when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg); - MessageExt lastMsg = mock(MessageExt.class); - when(lastMsg.getStoreTimestamp()).thenReturn(300L); - when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg); - - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, null); - MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); - requestBody.setFromQueueRouteData(fromQueueRouteData); - LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(); - toQueueRouteData.setMessageQueue(new MessageQueue(topic, "toBroker", 1)); - requestBody.setToQueueRouteData(toQueueRouteData); - request.setBody(requestBody.encode()); - - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); - fromQueueRouteData.setState(MessageQueueRouteState.ReadOnly); - fromQueueRouteData.setOffsetMax(100L); - fromQueueRouteData.setFirstMsgTimeMillis(200L); - fromQueueRouteData.setLastMsgTimeMillis(300L); - toQueueRouteData.setLogicalQueueDelta(590L); - MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class); - assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData); - assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData); - assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData)); - } - - @Test - public void testMigrateTopicLogicalQueueCommit() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId); - LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(); - fromQueueRouteData.setMessageQueue(new MessageQueue(topic, "fromBroker", 0)); - LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(toQueueRouteData))); - - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, null); - MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); - requestBody.setFromQueueRouteData(fromQueueRouteData); - requestBody.setToQueueRouteData(toQueueRouteData); - request.setBody(requestBody.encode()); - - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); - MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class); - assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData); - assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData); - assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(toQueueRouteData)); - } - - @Test - public void testMigrateTopicLogicalQueueNotify() throws Exception { - int queueId = 0; - int logicalQueueIndex = 0; - TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager(); - LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 100L, new MessageQueue(topic, "fromBroker", queueId), MessageQueueRouteState.ReadOnly, 10L, 410L, 200L, 300L, brokerController.getBrokerAddr()); - LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, new MessageQueue(topic, "toBroker", queueId), MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData))); - - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, null); - MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); - requestBody.setFromQueueRouteData(fromQueueRouteData); - requestBody.setToQueueRouteData(toQueueRouteData); - request.setBody(requestBody.encode()); - - RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); - assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS); - assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData)); - } private RemotingCommand buildCreateTopicRequest(String topic) { CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java index b6377553d7..8d3f624899 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java @@ -18,33 +18,21 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.client.ClientChannelInfo; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext; import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; -import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; @@ -53,7 +41,6 @@ import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -61,15 +48,17 @@ import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import static java.util.Optional.ofNullable; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.intThat; -import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,94 +194,6 @@ public class PullMessageProcessorTest { assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_OFFSET_MOVED); } - @Test - public void testProcessRequest_LogicalQueue() throws Exception { - String brokerName = brokerController.getBrokerConfig().getBrokerName(); - int queueId = 1; - - GetMessageResult getMessageResult = createGetMessageResult(); - when(messageStore.getMessage(anyString(), eq(topic), eq(queueId), eq(456L), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult); - when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId))).thenReturn(2000L); - when(messageStore.getMinPhyOffset()).thenReturn(0L); - - LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic); - LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerName, queueId), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1)); - logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1); - - // normal - { - final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); - RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - } - // write only - queueRouteData1.setState(MessageQueueRouteState.WriteOnly); - { - final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); - RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND); - } - // no message and redirect - queueRouteData1.setState(MessageQueueRouteState.ReadOnly); - queueRouteData1.setOffsetMax(460); - queueRouteData1.setFirstMsgTimeMillis(100); - queueRouteData1.setLastMsgTimeMillis(200); - LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 460, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); - logicalQueuesInfo.get(0).add(queueRouteData2); - getMessageResult.setStatus(GetMessageStatus.OFFSET_FOUND_NULL); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(460L - 1L))).thenReturn(1000L); - { - final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); - RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND); - assertThat(response.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT); - } - // same message queue has two routes - queueRouteData2.setState(MessageQueueRouteState.ReadOnly); - queueRouteData2.setOffsetMax(50); - queueRouteData2.setFirstMsgTimeMillis(300); - queueRouteData2.setLastMsgTimeMillis(400); - LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(0, 510, new MessageQueue(topic, queueRouteData2.getBrokerName(), queueId), MessageQueueRouteState.Normal, 460, -1, -1, -1, queueRouteData1.getBrokerAddr()); - logicalQueuesInfo.get(0).add(queueRouteData3); - logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData3.getQueueId(), queueRouteData3); - { - GetMessageResult getMessageResult2 = createGetMessageResult(); - getMessageResult2.setStatus(GetMessageStatus.FOUND); - getMessageResult2.setNextBeginOffset(460); - when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), eq(4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2); - } - { - GetMessageResult getMessageResult2 = createGetMessageResult(); - getMessageResult2.setStatus(GetMessageStatus.FOUND); - getMessageResult2.setNextBeginOffset(470); - lenient().when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), intThat(i -> i > 4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2); - } - { - final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); - RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); - assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).doesNotContainKey(MessageConst.PROPERTY_REDIRECT); - PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader(); - assertThat(header.getNextBeginOffset()).isEqualTo(460); - } - { - when(messageStore.getMinPhyOffset()).thenReturn(100000L); - - final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE); - RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_RETRY_IMMEDIATELY); - assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).containsKey(MessageConst.PROPERTY_REDIRECT); - PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader(); - assertThat(header.getNextBeginOffset()).isEqualTo(460); - } - } - private RemotingCommand createPullMsgCommand(int requestCode) { PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); requestHeader.setCommitOffset(123L); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java index e768469f7a..b4ae34ddac 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java @@ -18,14 +18,7 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.broker.transaction.TransactionalMessageService; @@ -35,13 +28,10 @@ import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -55,7 +45,6 @@ import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -65,6 +54,12 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -229,62 +224,6 @@ public class SendMessageProcessorTest { assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS); } - - @Test - public void testProcessRequest_LogicalQueue() throws Exception { - when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class))) - .thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK)))); - - LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic); - LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()); - logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1)); - logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1); - - SendMessageRequestHeader requestHeader = createSendMsgRequestHeader(); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); - request.setBody(new byte[] {'a'}); - request.makeCustomHeaderToNet(); - - // normal - RemotingCommand responseToReturn; - { - CompletableFuture responseFuture = new CompletableFuture<>(); - doAnswer(invocation -> { - responseFuture.complete(invocation.getArgument(0)); - return null; - }).when(handlerContext).writeAndFlush(any(Object.class)); - - responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); - if (responseToReturn == null) { - responseToReturn = responseFuture.get(3, TimeUnit.SECONDS); - } - } - - assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); - assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); - - // read only - queueRouteData1.setState(MessageQueueRouteState.ReadOnly); - responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); - assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); - assertThat(responseToReturn.getRemark()).contains("not writable"); - - // read only and forward - logicalQueuesInfo.get(0).add(new LogicalQueueRouteData(0, 100, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr())); - - responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); - assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); - assertThat(responseToReturn.getRemark()).contains("forward error"); - - // read only and redirect - requestHeader = (SendMessageRequestHeader) request.readCustomHeader(); - requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); - request.makeCustomHeaderToNet(); - responseToReturn = sendMessageProcessor.processRequest(handlerContext, request); - assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION); - assertThat(responseToReturn.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT); - } - private RemotingCommand createSendTransactionMsgCommand(int requestCode) { SendMessageRequestHeader header = createSendMsgRequestHeader(); int sysFlag = header.getSysFlag(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java deleted file mode 100644 index f504965fad..0000000000 --- a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.broker.topic; - -import com.google.common.collect.Lists; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker; -import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.DataVersion; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.junit.MockitoJUnitRunner; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class TopicConfigManagerTest { - @Mock - private DefaultMessageStore messageStore; - @Mock - private BrokerController brokerController; - - private TopicConfigManager topicConfigManager; - - private static final String topic = "FooBar"; - private static final String broker1Name = "broker1"; - private static final String broker1Addr = "127.0.0.1:12345"; - private static final int queueId1 = 1; - private static final String broker2Name = "broker2"; - private static final String broker2Addr = "127.0.0.2:12345"; - private static final int queueId2 = 2; - - @Before - public void before() { - BrokerConfig brokerConfig = new BrokerConfig(); - brokerConfig.setBrokerName(broker1Name); - when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); - - when(brokerController.getMessageStore()).thenReturn(messageStore); - - MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); - messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir")); - when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); - - this.topicConfigManager = new TopicConfigManager(brokerController); - this.topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic)); - } - - @After - public void after() throws Exception { - if (topicConfigManager != null) { - Files.deleteIfExists(Paths.get(topicConfigManager.configFilePath())); - } - } - - @Test - public void logicalQueueCleanTest() { - LogicalQueuesInfoInBroker info = this.topicConfigManager.getOrCreateLogicalQueuesInfo(topic); - topicConfigManager.logicalQueueClean(); - assertThat(info).isEmpty(); - - final int logicalQueueIndex = 0; - LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(logicalQueueIndex, 0, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker1Addr); - List l = Lists.newArrayList(new LogicalQueueRouteData(queueRouteData1)); - info.put(logicalQueueIndex, l); - - topicConfigManager.logicalQueueClean(); - assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData1)); - verify(messageStore, never()).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), anyLong()); - verify(messageStore, never()).getMinPhyOffset(); - verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); - - LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(logicalQueueIndex, 100, new MessageQueue(topic, broker2Name, queueId2), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr); - l.add(new LogicalQueueRouteData(queueRouteData2)); - queueRouteData1 = l.get(0); - queueRouteData1.setState(MessageQueueRouteState.ReadOnly); - queueRouteData1.setOffsetMax(100); - queueRouteData1.setFirstMsgTimeMillis(200); - queueRouteData1.setLastMsgTimeMillis(300); - queueRouteData1 = new LogicalQueueRouteData(queueRouteData1); - LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(logicalQueueIndex, 200, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 100, -1, -1, -1, broker1Addr); - l.add(new LogicalQueueRouteData(queueRouteData3)); - queueRouteData2 = l.get(1); - queueRouteData2.setState(MessageQueueRouteState.ReadOnly); - queueRouteData2.setOffsetMax(100); - queueRouteData2.setFirstMsgTimeMillis(400); - queueRouteData2.setLastMsgTimeMillis(500); - queueRouteData2 = new LogicalQueueRouteData(queueRouteData2); - when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1))).thenReturn(1000L); - when(messageStore.getMinPhyOffset()).thenReturn(0L); - topicConfigManager.logicalQueueClean(); - assertThat(info.get(logicalQueueIndex)).isEqualTo(Arrays.asList(queueRouteData1, queueRouteData2, queueRouteData3)); - verify(messageStore).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1)); - verify(messageStore).getMinPhyOffset(); - verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); - - when(messageStore.getMinPhyOffset()).thenReturn(2000L); - topicConfigManager.logicalQueueClean(); - assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData3)); - verify(brokerController).registerIncrementBrokerData(ArgumentMatchers.argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class)); - } -} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java new file mode 100644 index 0000000000..6b4faab5d5 --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.broker.topic; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.common.statictopic.TopicRemappingDetailWrapper; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class TopicQueueMappingManagerTest { + @Mock + private BrokerController brokerController; + private static final String broker1Name = "broker1"; + + @Before + public void before() { + BrokerConfig brokerConfig = new BrokerConfig(); + brokerConfig.setBrokerName(broker1Name); + when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir")); + messageStoreConfig.setDeleteWhen("01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23;00"); + when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); + } + + + private void delete(TopicQueueMappingManager topicQueueMappingManager) throws Exception { + if (topicQueueMappingManager == null) { + return; + } + Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath())); + Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath() + ".bak")); + + + } + + @Test + public void testEncodeDecode() throws Exception { + Map mappingDetailMap = new HashMap<>(); + TopicQueueMappingManager topicQueueMappingManager = null; + Set brokers = new HashSet(); + brokers.add(broker1Name); + { + for (int i = 0; i < 10; i++) { + String topic = UUID.randomUUID().toString(); + int queueNum = 10; + TopicRemappingDetailWrapper topicRemappingDetailWrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, brokers, new HashMap<>()); + Assert.assertEquals(1, topicRemappingDetailWrapper.getBrokerConfigMap().size()); + TopicQueueMappingDetail topicQueueMappingDetail = topicRemappingDetailWrapper.getBrokerConfigMap().values().iterator().next().getMappingDetail(); + Assert.assertEquals(queueNum, topicQueueMappingDetail.getHostedQueues().size()); + mappingDetailMap.put(topic, topicQueueMappingDetail); + } + } + + { + topicQueueMappingManager = new TopicQueueMappingManager(brokerController); + Assert.assertTrue(topicQueueMappingManager.load()); + Assert.assertEquals(0, topicQueueMappingManager.getTopicQueueMappingTable().size()); + for (TopicQueueMappingDetail mappingDetail : mappingDetailMap.values()) { + for (int i = 0; i < 10; i++) { + topicQueueMappingManager.updateTopicQueueMapping(mappingDetail, false, false, true); + } + } + topicQueueMappingManager.persist(); + } + + { + topicQueueMappingManager = new TopicQueueMappingManager(brokerController); + Assert.assertTrue(topicQueueMappingManager.load()); + Assert.assertEquals(mappingDetailMap.size(), topicQueueMappingManager.getTopicQueueMappingTable().size()); + for (TopicQueueMappingDetail topicQueueMappingDetail: topicQueueMappingManager.getTopicQueueMappingTable().values()) { + Assert.assertEquals(topicQueueMappingDetail, mappingDetailMap.get(topicQueueMappingDetail.getTopic())); + } + } + delete(topicQueueMappingManager); + } +} \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index cf8cbb0d12..8a6340b95f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -691,7 +691,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { msg.setTopic(withNamespace(msg.getTopic())); - this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, null); + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, (String) null); } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java b/client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java index 30d995270c..d8875421b1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java @@ -26,6 +26,7 @@ public class PullResult { private final long maxOffset; private List msgFoundList; + public PullResult(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, List msgFoundList) { super(); diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java index 6b76238371..fc0b29c680 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -16,6 +16,20 @@ */ package org.apache.rocketmq.client.consumer.store; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.OffsetNotFoundException; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.exception.RemotingException; + import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -23,18 +37,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.FindBrokerResult; -import org.apache.rocketmq.client.impl.factory.MQClientInstance; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; -import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; -import org.apache.rocketmq.remoting.exception.RemotingException; /** * Remote storage implementation @@ -94,7 +96,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore { return brokerOffset; } // No offset in broker - catch (MQBrokerException e) { + catch (OffsetNotFoundException e) { return -1; } //Other exceptions @@ -108,7 +110,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore { } } - return -1; + return -3; } @Override @@ -199,10 +201,10 @@ public class RemoteBrokerOffsetStore implements OffsetStore { @Override public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); } if (findBrokerResult != null) { @@ -226,11 +228,11 @@ public class RemoteBrokerOffsetStore implements OffsetStore { private long fetchConsumeOffsetFromBroker(MessageQueue mq) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); } if (findBrokerResult != null) { diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java b/client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java similarity index 59% rename from common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java rename to client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java index fa8d50d419..e73bbbf150 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java +++ b/client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java @@ -14,24 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.common.protocol.header; +package org.apache.rocketmq.client.exception; -import org.apache.rocketmq.remoting.CommandCustomHeader; -import org.apache.rocketmq.remoting.annotation.CFNotNull; -import org.apache.rocketmq.remoting.exception.RemotingCommandException; +public class OffsetNotFoundException extends MQBrokerException { -public class DeleteTopicLogicalQueueRequestHeader implements CommandCustomHeader { - @CFNotNull - private String topic; - - @Override public void checkFields() throws RemotingCommandException { + public OffsetNotFoundException() { } - public String getTopic() { - return topic; + public OffsetNotFoundException(int responseCode, String errorMessage) { + super(responseCode, errorMessage); } - public void setTopic(String topic) { - this.topic = topic; + public OffsetNotFoundException(int responseCode, String errorMessage, String brokerAddr) { + super(responseCode, errorMessage, brokerAddr); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index dce830c7b9..0a6e005222 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -16,41 +16,6 @@ */ package org.apache.rocketmq.client.impl; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; -import org.apache.rocketmq.client.QueryResult; -import org.apache.rocketmq.client.Validators; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.exception.MQRedirectException; -import org.apache.rocketmq.client.impl.factory.MQClientInstance; -import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.message.MessageClientIDSetter; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageId; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.NamespaceUtil; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; -import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.remoting.InvokeCallback; -import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.apache.rocketmq.remoting.netty.ResponseFuture; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; - import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -62,6 +27,36 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.rocketmq.client.QueryResult; +import org.apache.rocketmq.client.Validators; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageId; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + public class MQAdminImpl { private final InternalLogger log = ClientLogger.getLog(); @@ -187,21 +182,16 @@ public class MQAdminImpl { } public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { - LogicalQueueRouteData logicalQueueRouteData = searchLogicalQueueRouteByTimestamp(mq, timestamp); - if (logicalQueueRouteData != null) { - mq = logicalQueueRouteData.getMessageQueue(); - } - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); } if (brokerAddr != null) { try { - long offset = this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp, + return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp, timeoutMillis); - return correctLogicalQueueOffset(offset, logicalQueueRouteData); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -211,60 +201,15 @@ public class MQAdminImpl { } public long maxOffset(MessageQueue mq) throws MQClientException { - return this.maxOffset(mq, true); - } - - public long maxOffset(MessageQueue mq, boolean committed) throws MQClientException { - final MessageQueue origMq = mq; - String topic = mq.getTopic(); - LogicalQueueRouteData previousQueueRouteData = null; - for (int i = 0; i < 5; i++) { - LogicalQueueRouteData maxQueueRouteData = this.searchLogicalQueueRouteByOffset(origMq, Long.MAX_VALUE); - if (maxQueueRouteData != null) { - if (previousQueueRouteData != null && Objects.equal(previousQueueRouteData.getMessageQueue(), maxQueueRouteData.getMessageQueue())) { - throw new MQClientException("Topic route info not latest", null); - } - previousQueueRouteData = maxQueueRouteData; - mq = maxQueueRouteData.getMessageQueue(); - } - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - if (null == brokerAddr) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - } - - if (brokerAddr != null) { - try { - long offset = this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, topic, mq.getQueueId(), committed, maxQueueRouteData != null, timeoutMillis); - return correctLogicalQueueOffset(offset, maxQueueRouteData); - } catch (MQRedirectException e) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, false, null, Collections.singleton(mq.getQueueId())); - continue; - } catch (Exception e) { - throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); - } - } - throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); - } - throw new MQClientException("Redirect exceed max times", null); - } - - public long minOffset(MessageQueue mq) throws MQClientException { - LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L); - if (minQueueRouteData != null) { - mq = minQueueRouteData.getMessageQueue(); - } - - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); } if (brokerAddr != null) { try { - long offset = this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); - return correctLogicalQueueOffset(offset, minQueueRouteData); + return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); } catch (Exception e) { throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } @@ -273,33 +218,29 @@ public class MQAdminImpl { throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } - private List queryLogicalQueueRouteData(MessageQueue mq) { - if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { - TopicRouteData topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic()); - if (topicRouteData == null) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic()); - } - if (topicRouteData != null) { - LogicalQueuesInfo logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); - if (logicalQueuesInfo != null) { - return logicalQueuesInfo.get(mq.getQueueId()); - } + public long minOffset(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis); + } catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); } } - return null; + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { - LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L); - if (minQueueRouteData != null) { - mq = minQueueRouteData.getMessageQueue(); - } - - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); if (null == brokerAddr) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq)); } if (brokerAddr != null) { @@ -504,71 +445,4 @@ public class MQAdminImpl { throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "The topic[" + topic + "] not matched route info"); } - - private static long correctLogicalQueueOffset(long offset, LogicalQueueRouteData logicalQueueRouteData) { - if (logicalQueueRouteData == null) { - return offset; - } - return logicalQueueRouteData.toLogicalQueueOffset(offset); - } - - private LogicalQueueRouteData searchLogicalQueueRouteByTimestamp(MessageQueue mq, long timestamp) { - List queueRouteDataList = this.queryLogicalQueueRouteData(mq); - if (queueRouteDataList == null) { - return null; - } - LogicalQueueRouteData logicalQueueRouteData = null; - for (LogicalQueueRouteData el : queueRouteDataList) { - if (!el.isReadable()) { - continue; - } - if (logicalQueueRouteData == null && el.getFirstMsgTimeMillis() < 0) { - logicalQueueRouteData = el; - } else if (el.getFirstMsgTimeMillis() >= 0) { - if (el.getFirstMsgTimeMillis() <= timestamp && el.getLastMsgTimeMillis() >= timestamp) { - logicalQueueRouteData = el; - break; - } - } - } - if (logicalQueueRouteData == null) { - logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); - } - return logicalQueueRouteData; - } - - private LogicalQueueRouteData searchLogicalQueueRouteByOffset(MessageQueue mq, long offset) { - List queueRouteDataList = this.queryLogicalQueueRouteData(mq); - if (queueRouteDataList == null) { - return null; - } - { - List list = Lists.newArrayListWithCapacity(queueRouteDataList.size()); - for (LogicalQueueRouteData queueRouteData : queueRouteDataList) { - if (LogicalQueueRouteData.READABLE_PREDICT.apply(queueRouteData)) { - list.add(queueRouteData); - } - } - queueRouteDataList = list; - } - if (queueRouteDataList.isEmpty()) { - return null; - } - if (offset <= 0) { - // min - return Collections.min(queueRouteDataList); - } else if (offset == Long.MAX_VALUE) { - // max - return Collections.max(queueRouteDataList); - } - Collections.sort(queueRouteDataList); - LogicalQueueRouteData searchKey = new LogicalQueueRouteData(); - searchKey.setLogicalQueueDelta(offset); - int idx = Collections.binarySearch(queueRouteDataList, searchKey); - if (idx < 0) { - idx = -idx - 1; - idx -= 1; - } - return queueRouteDataList.get(idx); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 114fd0cf73..9823e817fc 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -16,7 +16,6 @@ */ package org.apache.rocketmq.client.impl; -import com.google.common.base.Function; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -42,7 +41,7 @@ import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.exception.MQRedirectException; +import org.apache.rocketmq.client.exception.OffsetNotFoundException; import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.factory.MQClientInstance; @@ -82,13 +81,11 @@ import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; import org.apache.rocketmq.common.protocol.body.ConsumeStatsList; import org.apache.rocketmq.common.protocol.body.ConsumerConnection; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; -import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.GetConsumerStatusBody; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody; import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody; @@ -98,13 +95,10 @@ import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.body.ResetOffsetBody; import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody; -import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody; -import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; -import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody; import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader; @@ -115,7 +109,6 @@ import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeade import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; -import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader; import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; @@ -149,7 +142,6 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHea import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; -import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; @@ -176,13 +168,10 @@ import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerR import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.InvokeCallback; @@ -201,8 +190,6 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -import static com.google.common.base.Optional.fromNullable; - public class MQClientAPIImpl { private final static InternalLogger log = ClientLogger.getLog(); @@ -335,7 +322,7 @@ public class MQClientAPIImpl { public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final long timeoutMillis) - throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + throws RemotingException, InterruptedException, MQClientException { CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); requestHeader.setTopic(topicConfig.getTopicName()); requestHeader.setDefaultTopic(defaultTopic); @@ -624,13 +611,9 @@ public class MQClientAPIImpl { producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); } catch (Exception e) { - if (e instanceof MQRedirectException) { - sendCallback.onException(e); - } else { - producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); - onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, - retryTimesWhenSendFailed, times, e, context, false, producer); - } + producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); + onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, + retryTimesWhenSendFailed, times, e, context, false, producer); } } else { producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); @@ -672,7 +655,7 @@ public class MQClientAPIImpl { String retryBrokerName = brokerName;//by default, it will send to the same broker if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName); - retryBrokerName = mqChosen.getBrokerName(); + retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen); } String addr = instance.findBrokerAddressInPublish(retryBrokerName); log.warn(String.format("async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}", tmp, msg.getTopic(), addr, @@ -709,11 +692,6 @@ public class MQClientAPIImpl { final RemotingCommand response, final String addr ) throws MQBrokerException, RemotingCommandException { - HashMap extFields = response.getExtFields(); - if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { - throw new MQRedirectException(response.getBody()); - } - SendStatus sendStatus; switch (response.getCode()) { case ResponseCode.FLUSH_DISK_TIMEOUT: { @@ -962,11 +940,6 @@ public class MQClientAPIImpl { private PullResult processPullResponse( final RemotingCommand response, final String addr) throws MQBrokerException, RemotingCommandException { - HashMap extFields = response.getExtFields(); - if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { - throw new MQRedirectException(response.getBody()); - } - PullStatus pullStatus = PullStatus.NO_NEW_MSG; switch (response.getCode()) { case ResponseCode.SUCCESS: @@ -981,6 +954,7 @@ public class MQClientAPIImpl { case ResponseCode.PULL_OFFSET_MOVED: pullStatus = PullStatus.OFFSET_ILLEGAL; break; + default: throw new MQBrokerException(response.getCode(), response.getRemark(), addr); } @@ -989,7 +963,7 @@ public class MQClientAPIImpl { (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(), - responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody()); + responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta()); } private PopResult processPopResponse(final String brokerName, final RemotingCommand response, String topic, @@ -1132,28 +1106,15 @@ public class MQClientAPIImpl { } public long getMaxOffset(final String addr, final String topic, final int queueId, final long timeoutMillis) - throws RemotingException, MQBrokerException, InterruptedException { - return getMaxOffset(addr, topic, queueId, true, false, timeoutMillis); - } - - public long getMaxOffset(final String addr, final String topic, final int queueId, boolean committed, - boolean fromLogicalQueue, - final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); requestHeader.setTopic(topic); requestHeader.setQueueId(queueId); - requestHeader.setCommitted(committed); - requestHeader.setLogicalQueue(fromLogicalQueue); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), request, timeoutMillis); assert response != null; - HashMap extFields = response.getExtFields(); - if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) { - throw new MQRedirectException(response.getBody()); - } switch (response.getCode()) { case ResponseCode.SUCCESS: { GetMaxOffsetResponseHeader responseHeader = @@ -1258,9 +1219,11 @@ public class MQClientAPIImpl { case ResponseCode.SUCCESS: { QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); - return responseHeader.getOffset(); } + case ResponseCode.PULL_NOT_FOUND: { + throw new OffsetNotFoundException(response.getCode(), response.getRemark(), addr); + } default: break; } @@ -1649,15 +1612,8 @@ public class MQClientAPIImpl { public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis, boolean allowTopicNotExist) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return getTopicRouteInfoFromNameServer(topic, timeoutMillis, allowTopicNotExist, null); - } - - public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis, - boolean allowTopicNotExist, Set logicalQueueIdsFilter) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); requestHeader.setTopic(topic); - requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG); - requestHeader.setLogicalQueueIdsFilter(logicalQueueIdsFilter); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader); @@ -1674,11 +1630,7 @@ public class MQClientAPIImpl { case ResponseCode.SUCCESS: { byte[] body = response.getBody(); if (body != null) { - return fromNullable(RemotingSerializable.decode(body, TopicRouteDataNameSrv.class)).transform(new Function() { - @Override public TopicRouteData apply(TopicRouteDataNameSrv srv) { - return srv.toTopicRouteData(); - } - }).orNull(); + return TopicRouteData.decode(body, TopicRouteData.class); } } default: @@ -2609,147 +2561,58 @@ public class MQClientAPIImpl { } } - public LogicalQueuesInfo queryTopicLogicalQueue(String brokerAddr, String topic, - long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader(); - requestHeader.setTopic(topic); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - return RemotingSerializable.decode(response.getBody(), LogicalQueuesInfo.class); - } - public void updateTopicLogicalQueue(String brokerAddr, String topic, int queueId, int logicalQueueIndex, - long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null); - UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody(); - requestBody.setTopic(topic); - requestBody.setQueueId(queueId); - requestBody.setLogicalQueueIdx(logicalQueueIndex); - request.setBody(requestBody.encode()); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - } - - public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic, long timeoutMillis) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader(); - requestHeader.setTopic(topic); - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - } - - public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null); - SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody(); - MessageQueue messageQueue = queueRouteData.getMessageQueue(); - requestBody.setTopic(messageQueue.getTopic()); - requestBody.setQueueId(messageQueue.getQueueId()); - requestBody.setLogicalQueueIndex(queueRouteData.getLogicalQueueIndex()); - request.setBody(requestBody.encode()); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); - } - - public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, - int logicalQueueIdx, - MessageQueueRouteState messageQueueRouteState, long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null); - ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody(); - requestBody.setTopic(topic); - requestBody.setQueueId(queueId); - requestBody.setLogicalQueueIndex(logicalQueueIdx); - requestBody.setMessageQueueRouteState(messageQueueRouteState); - request.setBody(requestBody.encode()); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); - } - - public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, int logicalQueueIdx, - MessageQueueRouteState messageQueueStatus, - long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null); - CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody(); - requestBody.setTopic(topic); - requestBody.setLogicalQueueIndex(logicalQueueIdx); - requestBody.setMessageQueueStatus(messageQueueStatus); - request.setBody(requestBody.encode()); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class); - } - - private MigrateLogicalQueueBody migrateTopicLogicalQueue(int requestCode, String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, - long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, null); - MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody(); - requestBody.setFromQueueRouteData(fromQueueRouteData); - requestBody.setToQueueRouteData(toQueueRouteData); - request.setBody(requestBody.encode()); - RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis); - assert response != null; - if (response.getCode() != ResponseCode.SUCCESS) { - throw new MQBrokerException(response.getCode(), response.getRemark()); - } - return response.getBody() != null ? RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class) : null; - } - - public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, - long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - - public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData, - long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - - public void migrateTopicLogicalQueueNotify(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData, - long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - - public TopicConfig getTopicConfig(final String brokerAddr, String topic, + public TopicConfigAndQueueMapping getTopicConfig(final String brokerAddr, String topic, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader(); header.setTopic(topic); + header.setLo(true); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, header); RemotingCommand response = this.remotingClient .invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis); assert response != null; switch (response.getCode()) { case ResponseCode.SUCCESS: { - return RemotingSerializable.decode(response.getBody(), TopicConfig.class); + return RemotingSerializable.decode(response.getBody(), TopicConfigAndQueueMapping.class); + } + //should check the exist + case ResponseCode.TOPIC_NOT_EXIST: { + //should return null? + break; } default: break; } throw new MQBrokerException(response.getCode(), response.getRemark()); } + + public void createStaticTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final TopicQueueMappingDetail topicQueueMappingDetail, boolean force, + final long timeoutMillis) throws RemotingException, InterruptedException, MQBrokerException { + CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); + requestHeader.setTopic(topicConfig.getTopicName()); + requestHeader.setDefaultTopic(defaultTopic); + requestHeader.setReadQueueNums(topicConfig.getReadQueueNums()); + requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums()); + requestHeader.setPerm(topicConfig.getPerm()); + requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name()); + requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag()); + requestHeader.setOrder(topicConfig.isOrder()); + requestHeader.setForce(force); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC, requestHeader); + request.setBody(topicQueueMappingDetail.encode()); + + RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr), + request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java index 702aef6c6a..ce9355ff24 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java @@ -306,7 +306,7 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService // Wrap topic with namespace before sending back message. msg.setTopic(this.defaultMQPushConsumer.withNamespace(msg.getTopic())); try { - this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName()); + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, this.defaultMQPushConsumer.queueWithNamespace(context.getMessageQueue())); return true; } catch (Exception e) { log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 4822d7d28b..845b158e17 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -16,25 +16,6 @@ */ package org.apache.rocketmq.client.impl.consumer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.MessageQueueListener; @@ -74,6 +55,26 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final InternalLogger log = ClientLogger.getLog(); @@ -928,9 +929,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); - if (pullResult instanceof PullResultWithLogicalQueues) { - pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt(); - } if (!this.consumeMessageHookList.isEmpty()) { ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext(); consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index e54d9b6a34..6d47573ffc 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -265,9 +265,6 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); - if (pullResult instanceof PullResultWithLogicalQueues) { - pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt(); - } //If namespace is not null , reset Topic without namespace. this.resetTopic(pullResult.getMsgFoundList()); if (!this.consumeMessageHookList.isEmpty()) { @@ -574,10 +571,15 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway); } + @Deprecated public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { try { - String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) + String destBrokerName = brokerName; + if (destBrokerName != null && destBrokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) { + destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPullConsumer.queueWithNamespace(new MessageQueue(msg.getTopic(), msg.getBrokerName(), msg.getQueueId()))); + } + String brokerAddr = (null != destBrokerName) ? this.mQClientFactory.findBrokerAddressInPublish(destBrokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); if (UtilAll.isBlank(consumerGroup)) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 92ee92603c..2b504cc85c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -72,6 +72,7 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.PopProcessQueueInfo; @@ -297,6 +298,9 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { long offset = -1L; try { offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue()); + if (offset < 0) { + throw new MQClientException(ResponseCode.SYSTEM_ERROR, "Unexpected offset " + offset); + } } catch (Exception e) { this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e); @@ -704,35 +708,55 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); } + @Deprecated public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + sendMessageBack(msg, delayLevel, brokerName, null); + } + + public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + sendMessageBack(msg, delayLevel, null, mq); + } + + + private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, final MessageQueue mq) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { try { - String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) - : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); - this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, - this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); + if ((brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) + || (mq != null && mq.getBrokerName().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX))) { + sendMessageBackAsNormalMessage(msg); + } else { + String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) + : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, + this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); + } } catch (Exception e) { log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); - - Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); - - String originMsgId = MessageAccessor.getOriginMessageId(msg); - MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); - - newMsg.setFlag(msg.getFlag()); - MessageAccessor.setProperties(newMsg, msg.getProperties()); - MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); - MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); - MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); - MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED); - newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); - - this.mQClientFactory.getDefaultMQProducer().send(newMsg); + sendMessageBackAsNormalMessage(msg); } finally { msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace())); } } + private void sendMessageBackAsNormalMessage(MessageExt msg) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); + + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); + + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); + MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); + MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED); + newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); + + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } + void ackAsync(MessageExt message, String consumerGroup) { final String extraInfo = message.getProperty(MessageConst.PROPERTY_POP_CK); @@ -743,15 +767,21 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { long queueOffset = ExtraInfoUtil.getQueueOffset(extraInfoStrs); String topic = message.getTopic(); + String desBrokerName = brokerName; + if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) { + desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId))); + } + + FindBrokerResult - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true); } if (findBrokerResult == null) { - log.error("The broker[" + brokerName + "] not exist"); + log.error("The broker[" + desBrokerName + "] not exist"); return; } @@ -784,11 +814,17 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs); int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs); + + String desBrokerName = brokerName; + if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) { + desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId))); + } + FindBrokerResult - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true); } if (findBrokerResult != null) { ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader(); @@ -798,10 +834,11 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { requestHeader.setConsumerGroup(consumerGroup); requestHeader.setExtraInfo(extraInfo); requestHeader.setInvisibleTime(invisibleTime); + //here the broker should be polished this.mQClientFactory.getMQClientAPIImpl().changeInvisibleTimeAsync(brokerName, findBrokerResult.getBrokerAddr(), requestHeader, ASYNC_TIMEOUT, callback); return; } - throw new MQClientException("The broker[" + brokerName + "] not exist", null); + throw new MQClientException("The broker[" + desBrokerName + "] not exist", null); } public int getMaxReconsumeTimes() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 5868e30f2b..30e8439bc3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -16,15 +16,12 @@ */ package org.apache.rocketmq.client.impl.consumer; -import com.alibaba.fastjson.JSON; -import com.google.common.base.Objects; import org.apache.rocketmq.client.consumer.PopCallback; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.PullStatus; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.hook.FilterMessageContext; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -40,12 +37,8 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; -import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.logging.InternalLogger; @@ -53,16 +46,12 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Random; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import static com.google.common.base.Optional.fromNullable; - public class PullAPIWrapper { private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mQClientFactory; @@ -83,36 +72,13 @@ public class PullAPIWrapper { public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult, final SubscriptionData subscriptionData) { - final PullResultExt pullResultExt = (PullResultExt) pullResult; - - LogicalQueueRouteData queueRouteData = null; - PullResultWithLogicalQueues pullResultWithLogicalQueues = null; - if (pullResultExt instanceof PullResultWithLogicalQueues) { - pullResultWithLogicalQueues = (PullResultWithLogicalQueues) pullResultExt; - queueRouteData = pullResultWithLogicalQueues.getQueueRouteData(); - } - - if (queueRouteData != null) { - pullResultWithLogicalQueues.setOrigPullResultExt(new PullResultExt(pullResultExt.getPullStatus(), - queueRouteData.toLogicalQueueOffset(pullResultExt.getNextBeginOffset()), - queueRouteData.toLogicalQueueOffset(pullResultExt.getMinOffset()), - // although this maxOffset may not belong to this queue route, but the actual value must be a larger one, and since maxOffset here is not an accurate value, we just do it to make things simple. - queueRouteData.toLogicalQueueOffset(pullResultExt.getMaxOffset()), - pullResultExt.getMsgFoundList(), - pullResultExt.getSuggestWhichBrokerId(), - pullResultExt.getMessageBinary())); - } + PullResultExt pullResultExt = (PullResultExt) pullResult; this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId()); if (PullStatus.FOUND == pullResult.getPullStatus()) { ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); List msgList = MessageDecoder.decodes(byteBuffer); - if (queueRouteData != null) { - // prevent pulled data is out of current queue route, this happens when some commit log data is cleaned in the broker but still pull from it. - msgList = queueRouteData.filterMessages(msgList); - } - List msgListFilterAgain = msgList; if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) { msgListFilterAgain = new ArrayList(msgList.size()); @@ -143,8 +109,8 @@ public class PullAPIWrapper { Long.toString(pullResult.getMaxOffset())); msg.setBrokerName(mq.getBrokerName()); msg.setQueueId(mq.getQueueId()); - if (queueRouteData != null) { - msg.setQueueOffset(queueRouteData.toLogicalQueueOffset(msg.getQueueOffset())); + if (pullResultExt.getOffsetDelta() != null) { + msg.setQueueOffset(pullResultExt.getOffsetDelta() + msg.getQueueOffset()); } } @@ -153,7 +119,7 @@ public class PullAPIWrapper { pullResultExt.setMessageBinary(null); - return pullResultExt; + return pullResult; } public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) { @@ -182,72 +148,30 @@ public class PullAPIWrapper { } public PullResult pullKernelImpl( - MessageQueue mq, + final MessageQueue mq, final String subExpression, final String expressionType, final long subVersion, - long offset, + final long offset, final int maxNums, final int sysFlag, - long commitOffset, + final long commitOffset, final long brokerSuspendMaxTimeMillis, final long timeoutMillis, final CommunicationMode communicationMode, - PullCallback pullCallback + final PullCallback pullCallback ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { - LogicalQueueContext logicalQueueContext = new LogicalQueueContext(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback); - while (true) { - try { - MessageQueue messageQueue = logicalQueueContext.getModifiedMessageQueue(); - if (messageQueue == null) { - if (pullCallback != null) { - pullCallback.onSuccess(logicalQueueContext.getPullResult()); - return null; - } else { - return logicalQueueContext.getPullResult(); - } - } - PullResult pullResult = this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, logicalQueueContext.getModifiedOffset(), maxNums, sysFlag, logicalQueueContext.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, logicalQueueContext.wrapPullCallback()); - return logicalQueueContext.wrapPullResult(pullResult); - } catch (MQRedirectException e) { - if (!logicalQueueContext.shouldRetry(e)) { - throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); - } - } - } - } else { - return this.pullKernelImplWithoutRetry(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback); - } - } - - public PullResult pullKernelImplWithoutRetry( - MessageQueue mq, - final String subExpression, - final String expressionType, - final long subVersion, - long offset, - final int maxNums, - final int sysFlag, - long commitOffset, - final long brokerSuspendMaxTimeMillis, - final long timeoutMillis, - final CommunicationMode communicationMode, - PullCallback pullCallback - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - String topic = mq.getTopic(); - int queueId = mq.getQueueId(); - FindBrokerResult findBrokerResult = - this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), this.recalculatePullFromWhichNode(mq), false); if (null == findBrokerResult) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); findBrokerResult = - this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), this.recalculatePullFromWhichNode(mq), false); } + if (findBrokerResult != null) { { // check version @@ -265,8 +189,8 @@ public class PullAPIWrapper { PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); requestHeader.setConsumerGroup(this.consumerGroup); - requestHeader.setTopic(topic); - requestHeader.setQueueId(queueId); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setQueueId(mq.getQueueId()); requestHeader.setQueueOffset(offset); requestHeader.setMaxMsgNums(maxNums); requestHeader.setSysFlag(sysFlagInner); @@ -278,15 +202,18 @@ public class PullAPIWrapper { String brokerAddr = findBrokerResult.getBrokerAddr(); if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { - brokerAddr = computePullFromWhichFilterServer(topic, brokerAddr); + brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr); } - return this.mQClientFactory.getMQClientAPIImpl().pullMessage( + + PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage( brokerAddr, requestHeader, timeoutMillis, communicationMode, pullCallback); + + return pullResult; } throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); @@ -373,10 +300,10 @@ public class PullAPIWrapper { public void popAsync(MessageQueue mq, long invisibleTime, int maxNums, String consumerGroup, long timeout, PopCallback popCallback, boolean poll, int initMode, boolean order, String expressionType, String expression) throws MQClientException, RemotingException, InterruptedException { - FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true); } if (findBrokerResult != null) { PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); @@ -403,228 +330,4 @@ public class PullAPIWrapper { throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); } - private class LogicalQueueContext implements PullCallback { - private final MessageQueue mq; - private final String subExpression; - private final String expressionType; - private final long subVersion; - private final long offset; - private final int maxNums; - private final int sysFlag; - private final long commitOffset; - private final long brokerSuspendMaxTimeMillis; - private final long timeoutMillis; - private final CommunicationMode communicationMode; - private final PullCallback pullCallback; - - private volatile LogicalQueuesInfo logicalQueuesInfo; - private volatile LogicalQueueRouteData logicalQueueRouteData; - private volatile boolean isMaxReadableQueueRoute; - - private volatile PullResultExt pullResult = null; - - private final AtomicInteger retry = new AtomicInteger(); - - public LogicalQueueContext(MessageQueue mq, String subExpression, String expressionType, long subVersion, - long offset, int maxNums, int sysFlag, long commitOffset, long brokerSuspendMaxTimeMillis, - long timeoutMillis, CommunicationMode communicationMode, - PullCallback pullCallback) { - this.mq = mq; - this.subExpression = subExpression; - this.expressionType = expressionType; - this.subVersion = subVersion; - this.offset = offset; - this.maxNums = maxNums; - this.sysFlag = sysFlag; - this.commitOffset = commitOffset; - this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis; - this.timeoutMillis = timeoutMillis; - this.communicationMode = communicationMode; - this.pullCallback = pullCallback; - - this.buildLogicalQueuesInfo(); - } - - private boolean notUsingLogicalQueue() { - return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null; - } - - private void buildLogicalQueuesInfo() { - TopicRouteData topicRouteData = PullAPIWrapper.this.mQClientFactory.queryTopicRouteData(mq.getTopic()); - if (topicRouteData != null) { - this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); - } - } - - @Override public void onSuccess(PullResult pullResult) { - this.pullCallback.onSuccess(this.wrapPullResult(pullResult)); - } - - @Override public void onException(Throwable t) { - if (!this.shouldRetry(t)) { - this.pullCallback.onException(t); - return; - } - MessageQueue messageQueue = this.getModifiedMessageQueue(); - if (messageQueue == null) { - this.pullCallback.onSuccess(this.getPullResult()); - return; - } - try { - PullAPIWrapper.this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, this.getModifiedOffset(), maxNums, sysFlag, this.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, this); - } catch (Exception e) { - this.pullCallback.onException(e); - } - } - - public MessageQueue getModifiedMessageQueue() { - if (this.notUsingLogicalQueue()) { - return this.mq; - } - this.logicalQueuesInfo.readLock().lock(); - try { - List queueRouteDataList = fromNullable(this.logicalQueuesInfo.get(this.mq.getQueueId())).or(Collections.emptyList()); - LogicalQueueRouteData searchKey = new LogicalQueueRouteData(); - searchKey.setState(MessageQueueRouteState.Normal); - searchKey.setLogicalQueueDelta(offset); - // it's sorted after getTopicRouteInfoFromNameServer - int startIdx = Collections.binarySearch(queueRouteDataList, searchKey); - if (startIdx < 0) { - startIdx = -startIdx - 1; - // lower entry - startIdx -= 1; - } - this.logicalQueueRouteData = null; - this.pullResult = null; - LogicalQueueRouteData lastReadableLogicalQueueRouteData = null; // first item which delta > offset - LogicalQueueRouteData minReadableLogicalQueueRouteData = null; - LogicalQueueRouteData maxReadableLogicalQueueRouteData = null; - for (int i = 0, size = queueRouteDataList.size(); i < size; i++) { - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); - if (!queueRouteData.isReadable()) { - continue; - } - maxReadableLogicalQueueRouteData = queueRouteData; - if (minReadableLogicalQueueRouteData == null) { - minReadableLogicalQueueRouteData = queueRouteData; - if (i < startIdx) { - // must consider following `i++` operation when invoke `continue`, so decrease first - i = startIdx - 1; - continue; - } - } - if (queueRouteData.getLogicalQueueDelta() > offset) { - if (this.logicalQueueRouteData != null) { - if (this.logicalQueueRouteData.toLogicalQueueOffset(this.logicalQueueRouteData.getOffsetMax()) <= offset) { - this.logicalQueueRouteData = queueRouteData; - } - break; - } else { - if (lastReadableLogicalQueueRouteData == null) { - lastReadableLogicalQueueRouteData = queueRouteData; - } - } - } else { - this.logicalQueueRouteData = queueRouteData; - } - } - if (this.logicalQueueRouteData == null) { - if (lastReadableLogicalQueueRouteData != null) { - this.pullResult = new PullResultExt(PullStatus.OFFSET_ILLEGAL, lastReadableLogicalQueueRouteData.getLogicalQueueDelta(), minReadableLogicalQueueRouteData.getLogicalQueueDelta(), maxReadableLogicalQueueRouteData.getLogicalQueueDelta(), null, 0, null); - return null; - } else { - if (maxReadableLogicalQueueRouteData != null) { - this.logicalQueueRouteData = maxReadableLogicalQueueRouteData; - } else { - if (!queueRouteDataList.isEmpty()) { - this.logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); - } else { - pullResult = new PullResultExt(PullStatus.NO_NEW_MSG, 0, 0, 0, null, 0, null); - return null; - } - } - } - } - this.isMaxReadableQueueRoute = this.logicalQueueRouteData.isSameTo(maxReadableLogicalQueueRouteData); - return this.logicalQueueRouteData.getMessageQueue(); - } finally { - this.logicalQueuesInfo.readLock().unlock(); - } - } - - public PullResultExt getPullResult() { - return pullResult; - } - - public PullCallback wrapPullCallback() { - if (this.notUsingLogicalQueue()) { - return this.pullCallback; - } - if (!CommunicationMode.ASYNC.equals(this.communicationMode)) { - return this.pullCallback; - } - return this; - } - - public long getModifiedOffset() { - return this.logicalQueueRouteData.toMessageQueueOffset(this.offset); - } - - public long getModifiedCommitOffset() { - // TODO should this be modified too? If offset is not in current broker's range, how do we handle it? - return this.commitOffset; - } - - public void incrRetry() { - this.retry.incrementAndGet(); - } - - public boolean shouldRetry(Throwable t) { - this.incrRetry(); - if (this.retry.get() >= 3) { - return false; - } - if (t instanceof MQRedirectException) { - MQRedirectException e = (MQRedirectException) t; - this.processResponseBody(e.getBody()); - return true; - } - return false; - } - - public PullResult wrapPullResult(PullResult pullResult) { - if (pullResult == null) { - return null; - } - if (this.logicalQueueRouteData == null) { - return pullResult; - } - if (!this.isMaxReadableQueueRoute && PullStatus.NO_MATCHED_MSG.equals(pullResult.getPullStatus())) { - PullStatus status = PullStatus.OFFSET_ILLEGAL; - if (pullResult instanceof PullResultExt) { - PullResultExt pullResultExt = (PullResultExt) pullResult; - pullResult = new PullResultExt(status, pullResultExt.getNextBeginOffset(), pullResultExt.getMinOffset(), pullResultExt.getMaxOffset(), pullResultExt.getMsgFoundList(), pullResultExt.getSuggestWhichBrokerId(), pullResultExt.getMessageBinary()); - } else { - pullResult = new PullResult(status, pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList()); - } - } - // method PullAPIWrapper#processPullResult will modify queueOffset/nextBeginOffset/minOffset/maxOffset - return new PullResultWithLogicalQueues(pullResult, this.logicalQueueRouteData); - } - - public void processResponseBody(byte[] responseBody) { - log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.logicalQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null); - if (responseBody != null) { - try { - List queueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA); - this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), queueRouteDataList); - return; - } catch (Exception e) { - log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.logicalQueueRouteData, e); - } - } - PullAPIWrapper.this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic(), false, null, Collections.singleton(this.mq.getQueueId())); - this.buildLogicalQueuesInfo(); - } - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java index c34a68f9ab..45538eda8d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java @@ -25,11 +25,23 @@ public class PullResultExt extends PullResult { private final long suggestWhichBrokerId; private byte[] messageBinary; + private final Long offsetDelta; + public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, List msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary) { + this(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList, suggestWhichBrokerId, messageBinary, 0L); + } + + public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, + List msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary, final Long offsetDelta) { super(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList); this.suggestWhichBrokerId = suggestWhichBrokerId; this.messageBinary = messageBinary; + this.offsetDelta = offsetDelta; + } + + public Long getOffsetDelta() { + return offsetDelta; } public byte[] getMessageBinary() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java deleted file mode 100644 index 8b85e548ca..0000000000 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.impl.consumer; - -import java.util.List; -import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.client.consumer.PullStatus; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; - -public class PullResultWithLogicalQueues extends PullResultExt { - private PullResultExt origPullResultExt; - private final LogicalQueueRouteData queueRouteData; - - public PullResultWithLogicalQueues(PullResult pullResult, LogicalQueueRouteData floorQueueRouteData) { - super(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(), - pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getSuggestWhichBrokerId() : MixAll.MASTER_ID, - pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getMessageBinary() : null); - if (pullResult instanceof PullResultExt) { - this.origPullResultExt = (PullResultExt) pullResult; - } else { - this.origPullResultExt = new PullResultExt(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(), MixAll.MASTER_ID, null); - } - this.queueRouteData = floorQueueRouteData; - } - - public PullResult getOrigPullResultExt() { - return origPullResultExt; - } - - public LogicalQueueRouteData getQueueRouteData() { - return queueRouteData; - } - - public void setOrigPullResultExt(PullResultExt pullResultExt) { - this.origPullResultExt = pullResultExt; - } - - @Override public PullStatus getPullStatus() { - return origPullResultExt.getPullStatus(); - } - - @Override public long getNextBeginOffset() { - return origPullResultExt.getNextBeginOffset(); - } - - @Override public long getMinOffset() { - return origPullResultExt.getMinOffset(); - } - - @Override public long getMaxOffset() { - return origPullResultExt.getMaxOffset(); - } - - @Override public List getMsgFoundList() { - return origPullResultExt.getMsgFoundList(); - } - - @Override public void setMsgFoundList(List msgFoundList) { - origPullResultExt.setMsgFoundList(msgFoundList); - } - - @Override public byte[] getMessageBinary() { - return origPullResultExt.getMessageBinary(); - } - - @Override public void setMessageBinary(byte[] messageBinary) { - origPullResultExt.setMessageBinary(messageBinary); - } - - @Override public long getSuggestWhichBrokerId() { - return origPullResultExt.getSuggestWhichBrokerId(); - } - - @Override public String toString() { - return "PullResultWithLogicalQueues{" + - "origPullResultExt=" + origPullResultExt + - ", queueRouteData=" + queueRouteData + - '}'; - } -} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index ab0d885c82..5788d1cbe0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -76,7 +76,7 @@ public abstract class RebalanceImpl { } public void unlock(final MessageQueue mq, final boolean oneway) { - FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true); if (findBrokerResult != null) { UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); requestBody.setConsumerGroup(this.consumerGroup); @@ -141,7 +141,8 @@ public abstract class RebalanceImpl { continue; } - Set mqs = result.get(mq.getBrokerName()); + String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq); + Set mqs = result.get(destBrokerName); if (null == mqs) { mqs = new HashSet(); result.put(mq.getBrokerName(), mqs); @@ -154,7 +155,7 @@ public abstract class RebalanceImpl { } public boolean lock(final MessageQueue mq) { - FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true); if (findBrokerResult != null) { LockBatchRequestBody requestBody = new LockBatchRequestBody(); requestBody.setConsumerGroup(this.consumerGroup); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index 09d1521654..76803dd6aa 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.ConsumeInitMode; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; @@ -188,7 +189,8 @@ public class RebalancePushImpl extends RebalanceImpl { } } } else { - result = -1; + throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query consume offset from " + + "offset store"); } break; } @@ -197,9 +199,11 @@ public class RebalancePushImpl extends RebalanceImpl { if (lastOffset >= 0) { result = lastOffset; } else if (-1 == lastOffset) { + //the offset will be fixed by the OFFSET_ILLEGAL process result = 0L; } else { - result = -1; + throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query offset from offset " + + "store"); } break; } @@ -226,7 +230,8 @@ public class RebalancePushImpl extends RebalanceImpl { } } } else { - result = -1; + throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query offset from offset " + + "store"); } break; } @@ -235,6 +240,10 @@ public class RebalancePushImpl extends RebalanceImpl { break; } + if (result < 0) { + throw new MQClientException(ResponseCode.SYSTEM_ERROR, "Found unexpected result " + result); + } + return result; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 874b1f8e7d..0181951523 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -16,26 +16,6 @@ */ package org.apache.rocketmq.client.impl.factory; -import java.io.UnsupportedEncodingException; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.admin.MQAdminExtInner; @@ -77,8 +57,6 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.ProducerData; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.logging.InternalLogger; @@ -88,10 +66,32 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import java.io.UnsupportedEncodingException; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static org.apache.rocketmq.common.rpc.ClientMetadata.topicRouteData2EndpointsForStaticTopic; public class MQClientInstance { private final static long LOCK_TIMEOUT_MILLIS = 3000; - private final InternalLogger log = ClientLogger.getLog(); + private final static InternalLogger log = ClientLogger.getLog(); private final ClientConfig clientConfig; private final int instanceIndex; private final String clientId; @@ -103,6 +103,7 @@ public class MQClientInstance { private final MQClientAPIImpl mQClientAPIImpl; private final MQAdminImpl mQAdminImpl; private final ConcurrentMap topicRouteTable = new ConcurrentHashMap(); + private final ConcurrentMap> topicEndPointsTable = new ConcurrentHashMap<>(); private final Lock lockNamesrv = new ReentrantLock(); private final Lock lockHeartbeat = new ReentrantLock(); private final ConcurrentMap> brokerAddrTable = @@ -162,8 +163,12 @@ public class MQClientInstance { MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION), RemotingCommand.getSerializeTypeConfigInThisServer()); } + + + public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topic, final TopicRouteData route) { TopicPublishInfo info = new TopicPublishInfo(); + // TO DO should check the usage of raw route, it is better to remove such field info.setTopicRouteData(route); if (route.getOrderTopicConf() != null && route.getOrderTopicConf().length() > 0) { String[] brokers = route.getOrderTopicConf().split(";"); @@ -177,28 +182,13 @@ public class MQClientInstance { } info.setOrderTopic(true); - } else if (route.getOrderTopicConf() == null && route.getLogicalQueuesInfo() != null) { + } else if (route.getOrderTopicConf() == null + && route.getTopicQueueMappingByBroker() != null + && !route.getTopicQueueMappingByBroker().isEmpty()) { info.setOrderTopic(false); - List messageQueueList = info.getMessageQueueList(); - LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo(); - for (Map.Entry> entry : logicalQueueInfo.entrySet()) { - boolean someWritable = false; - for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) { - if (logicalQueueRouteData.isWritable()) { - someWritable = true; - break; - } - } - if (!someWritable) { - continue; - } - MessageQueue mq = new MessageQueue(); - mq.setQueueId(entry.getKey()); - mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - mq.setTopic(topic); - messageQueueList.add(mq); - } - Collections.sort(messageQueueList, new Comparator() { + ConcurrentMap mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route); + info.getMessageQueueList().addAll(mqEndPoints.keySet()); + Collections.sort(info.getMessageQueueList(), new Comparator() { @Override public int compare(MessageQueue o1, MessageQueue o2) { return MixAll.compareInteger(o1.getQueueId(), o2.getQueueId()); } @@ -239,26 +229,10 @@ public class MQClientInstance { public static Set topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) { Set mqList = new HashSet(); - if (route.getLogicalQueuesInfo() != null) { - LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo(); - for (Map.Entry> entry : logicalQueueInfo.entrySet()) { - boolean someReadable = false; - for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) { - if (logicalQueueRouteData.isReadable()) { - someReadable = true; - break; - } - } - if (!someReadable) { - continue; - } - MessageQueue mq = new MessageQueue(); - mq.setQueueId(entry.getKey()); - mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - mq.setTopic(topic); - mqList.add(mq); - } - return mqList; + if (route.getTopicQueueMappingByBroker() != null + && !route.getTopicQueueMappingByBroker().isEmpty()) { + ConcurrentMap mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route); + return mqEndPoints.keySet(); } List qds = route.getQueueDatas(); for (QueueData qd : qds) { @@ -656,11 +630,6 @@ public class MQClientInstance { public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, DefaultMQProducer defaultMQProducer) { - return this.updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQProducer, null); - } - - public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, - DefaultMQProducer defaultMQProducer, Set logicalQueueIdsFilter) { try { if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { @@ -676,7 +645,7 @@ public class MQClientInstance { } } } else { - topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout(), true, logicalQueueIdsFilter); + topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout()); } if (topicRouteData != null) { TopicRouteData old = this.topicRouteTable.get(topic); @@ -688,31 +657,19 @@ public class MQClientInstance { } if (changed) { - TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData); - if (logicalQueueIdsFilter != null && cloneTopicRouteData.getLogicalQueuesInfo() != null) { - TopicRouteData curTopicRouteData = this.topicRouteTable.get(topic); - if (curTopicRouteData != null) { - LogicalQueuesInfo curLogicalQueuesInfo = curTopicRouteData.getLogicalQueuesInfo(); - if (curLogicalQueuesInfo != null) { - LogicalQueuesInfo cloneLogicalQueuesInfo = cloneTopicRouteData.getLogicalQueuesInfo(); - curLogicalQueuesInfo.readLock().lock(); - try { - for (Entry> entry : curLogicalQueuesInfo.entrySet()) { - if (!cloneLogicalQueuesInfo.containsKey(entry.getKey())) { - cloneLogicalQueuesInfo.put(entry.getKey(), entry.getValue()); - } - } - } finally { - curLogicalQueuesInfo.readLock().unlock(); - } - } - } - } for (BrokerData bd : topicRouteData.getBrokerDatas()) { this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs()); } + // Update endpoint map + { + ConcurrentMap mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData); + if (mqEndPoints != null && !mqEndPoints.isEmpty()) { + topicEndPointsTable.put(topic, mqEndPoints); + } + } + // Update Pub info { TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData); @@ -739,6 +696,7 @@ public class MQClientInstance { } } } + TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData); log.info("topicRouteTable.put. Topic = {}, TopicRouteData[{}]", topic, cloneTopicRouteData); this.topicRouteTable.put(topic, cloneTopicRouteData); return true; @@ -865,13 +823,6 @@ public class MQClientInstance { private boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) { if (olddata == null || nowdata == null) return true; - LogicalQueuesInfo oldLogicalQueuesInfo = olddata.getLogicalQueuesInfo(); - LogicalQueuesInfo newLogicalQueuesInfo = nowdata.getLogicalQueuesInfo(); - if (oldLogicalQueuesInfo != null && newLogicalQueuesInfo != null) { - return oldLogicalQueuesInfo.keySet().equals(newLogicalQueuesInfo.keySet()); - } else if (oldLogicalQueuesInfo != null || newLogicalQueuesInfo != null) { - return true; - } TopicRouteData old = new TopicRouteData(olddata); TopicRouteData now = new TopicRouteData(nowdata); Collections.sort(old.getQueueDatas()); @@ -1067,7 +1018,21 @@ public class MQClientInstance { return this.consumerTable.get(group); } + + public String getBrokerNameFromMessageQueue(final MessageQueue mq) { + if (topicEndPointsTable != null + && topicEndPointsTable.get(mq.getTopic()) != null + && !topicEndPointsTable.get(mq.getTopic()).isEmpty()) { + return topicEndPointsTable.get(mq.getTopic()).get(mq); + } + return mq.getBrokerName(); + } + + public FindBrokerResult findBrokerAddressInAdmin(final String brokerName) { + if (brokerName == null) { + return null; + } String brokerAddr = null; boolean slave = false; boolean found = false; @@ -1097,7 +1062,11 @@ public class MQClientInstance { return null; } + public String findBrokerAddressInPublish(final String brokerName) { + if (brokerName == null) { + return null; + } HashMap map = this.brokerAddrTable.get(brokerName); if (map != null && !map.isEmpty()) { return map.get(MixAll.MASTER_ID); @@ -1106,11 +1075,15 @@ public class MQClientInstance { return null; } + public FindBrokerResult findBrokerAddressInSubscribe( final String brokerName, final long brokerId, final boolean onlyThisBroker ) { + if (brokerName == null) { + return null; + } String brokerAddr = null; boolean slave = false; boolean found = false; @@ -1141,7 +1114,7 @@ public class MQClientInstance { return null; } - public int findBrokerVersion(String brokerName, String brokerAddr) { + private int findBrokerVersion(String brokerName, String brokerAddr) { if (this.brokerVersionTable.containsKey(brokerName)) { if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) { return this.brokerVersionTable.get(brokerName).get(brokerAddr); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 00df89c85b..bf2ca28af1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -16,16 +16,12 @@ */ package org.apache.rocketmq.client.impl.producer; -import com.alibaba.fastjson.JSON; -import com.google.common.base.Objects; import java.io.IOException; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Random; import java.util.Set; import java.util.concurrent.BlockingQueue; @@ -45,7 +41,6 @@ import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.exception.MQRedirectException; import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.CheckForbiddenContext; import org.apache.rocketmq.client.hook.CheckForbiddenHook; @@ -67,7 +62,6 @@ import org.apache.rocketmq.client.producer.RequestFutureTable; import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.SendResultForLogicalQueue; import org.apache.rocketmq.client.producer.SendStatus; import org.apache.rocketmq.client.producer.TransactionCheckListener; import org.apache.rocketmq.client.producer.TransactionListener; @@ -88,13 +82,9 @@ import org.apache.rocketmq.common.message.MessageId; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.message.MessageType; import org.apache.rocketmq.common.protocol.NamespaceUtil; -import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.utils.CorrelationIdUtil; import org.apache.rocketmq.logging.InternalLogger; @@ -730,43 +720,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { final SendCallback sendCallback, final TopicPublishInfo topicPublishInfo, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) { - LogicalQueueSendContext logicalQueueContext = new LogicalQueueSendContext(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); - while (true) { - try { - SendResult sendResult = this.sendKernelImplWithoutRetry(msg, - logicalQueueContext.getModifiedMessageQueue(), - communicationMode, - logicalQueueContext.wrapSendCallback(), - topicPublishInfo, - timeout); - return logicalQueueContext.wrapSendResult(sendResult); - } catch (MQRedirectException e) { - if (!logicalQueueContext.shouldRetry(e)) { - throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); - } - } catch (RemotingException e) { - if (!logicalQueueContext.shouldRetry(e)) { - throw e; - } - } - } - } else { - return sendKernelImplWithoutRetry(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout); - } - } - - private SendResult sendKernelImplWithoutRetry(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + String brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName); if (null == brokerAddr) { tryToFindTopicPublishInfo(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName); } SendMessageContext context = null; @@ -798,10 +758,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; } - if (!CommunicationMode.ONEWAY.equals(communicationMode)) { - sysFlag |= MessageSysFlag.LOGICAL_QUEUE_FLAG; - } - if (hasCheckForbiddenHook()) { CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); @@ -890,7 +846,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( brokerAddr, - mq.getBrokerName(), + brokerName, tmpMessage, requestHeader, timeout - costTimeAsync, @@ -910,7 +866,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( brokerAddr, - mq.getBrokerName(), + brokerName, msg, requestHeader, timeout - costTimeSync, @@ -941,7 +897,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } - throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + throw new MQClientException("The broker[" + brokerName + "] not exist", null); } public MQClientInstance getmQClientFactory() { @@ -1042,7 +998,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { executeEndTransactionHook(context); } } - /** * DEFAULT ONEWAY ------------------------------------------------------- */ @@ -1377,7 +1332,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); } String transactionId = sendResult.getTransactionId(); - final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); + final String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(defaultMQProducer.queueWithNamespace(sendResult.getMessageQueue())); + final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(destBrokerName); EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); requestHeader.setTransactionId(transactionId); requestHeader.setCommitLogOffset(id.getOffset()); @@ -1682,178 +1638,4 @@ public class DefaultMQProducerImpl implements MQProducerInner { public DefaultMQProducer getDefaultMQProducer() { return defaultMQProducer; } - - private class LogicalQueueSendContext implements SendCallback { - private final Message msg; - private final MessageQueue mq; - private final CommunicationMode communicationMode; - private final SendCallback sendCallback; - private final TopicPublishInfo topicPublishInfo; - private final long timeout; - - private volatile LogicalQueuesInfo logicalQueuesInfo; - private volatile LogicalQueueRouteData writableQueueRouteData; - - private final AtomicInteger retry = new AtomicInteger(); - - public LogicalQueueSendContext(Message msg, MessageQueue mq, - CommunicationMode communicationMode, SendCallback sendCallback, - TopicPublishInfo topicPublishInfo, long timeout) { - this.msg = msg; - this.mq = mq; - this.communicationMode = communicationMode; - this.sendCallback = sendCallback; - this.topicPublishInfo = topicPublishInfo; - this.timeout = timeout; - - if (topicPublishInfo == null) { - topicPublishInfo = DefaultMQProducerImpl.this.tryToFindTopicPublishInfo(mq.getTopic()); - } - if (topicPublishInfo != null) { - this.logicalQueuesInfo = topicPublishInfo.getTopicRouteData().getLogicalQueuesInfo(); - } else { - this.logicalQueuesInfo = null; - } - } - - private boolean notUsingLogicalQueue() { - return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null; - } - - public MessageQueue getModifiedMessageQueue() throws MQClientException { - if (this.notUsingLogicalQueue()) { - return this.mq; - } - this.writableQueueRouteData = getWritableQueueRouteData(); - MessageQueue mq = new MessageQueue(this.mq); - mq.setBrokerName(writableQueueRouteData.getBrokerName()); - mq.setQueueId(writableQueueRouteData.getQueueId()); - return mq; - } - - private LogicalQueueRouteData getWritableQueueRouteData() throws MQClientException { - this.logicalQueuesInfo.readLock().lock(); - try { - List queueRouteDataList = logicalQueuesInfo.get(mq.getQueueId()); - if (queueRouteDataList == null || queueRouteDataList.size() == 0) { - throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no queue route data found", mq.getQueueId()), null); - } - // usually writable queue is placed in the last position, or second last when queue migrating - for (int i = queueRouteDataList.size() - 1; i >= 0; i--) { - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i); - if (queueRouteData.isWritable()) { - return queueRouteData; - } - } - throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no writable queue route data found", mq.getQueueId()), null); - } finally { - this.logicalQueuesInfo.readLock().unlock(); - } - } - - @Override public void onSuccess(SendResult sendResult) { - this.sendCallback.onSuccess(this.wrapSendResult(sendResult)); - } - - @Override public void onException(Throwable t) { - if (this.shouldRetry(t)) { - try { - DefaultMQProducerImpl.this.sendKernelImplWithoutRetry(msg, this.getModifiedMessageQueue(), communicationMode, this, topicPublishInfo, timeout); - return; - } catch (Exception e) { - t = e; - } - } - if (t instanceof MQRedirectException) { - t = new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect"); - } - this.sendCallback.onException(t); - } - - private void handleRedirectException(MQRedirectException re) { - byte[] responseBody = re.getBody(); - log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.writableQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null); - - try { - List newQueueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA); - this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), newQueueRouteDataList); - } catch (Exception e) { - log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.writableQueueRouteData, e); - DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId())); - TopicRouteData topicRouteData = DefaultMQProducerImpl.this.mQClientFactory.getAnExistTopicRouteData(mq.getTopic()); - if (topicRouteData != null) { - this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo(); - } else { - this.logicalQueuesInfo = null; - } - } - } - - public SendCallback wrapSendCallback() { - if (this.notUsingLogicalQueue()) { - return this.sendCallback; - } - if (!CommunicationMode.ASYNC.equals(this.communicationMode)) { - return this.sendCallback; - } - return this; - } - - public boolean shouldRetry(Throwable t) { - this.incrRetry(); - if (this.exceedMaxRetry()) { - log.warn("retry {} too many times: {}", this.retry.get(), this.writableQueueRouteData); - return false; - } - if (!this.writableQueueRouteData.isWritable()) { - log.warn("no writable queue: {}", this.writableQueueRouteData); - return false; - } - if (t instanceof MQRedirectException) { - this.handleRedirectException((MQRedirectException) t); - return true; - } - return !(t instanceof RemotingException) || this.handleRemotingException((RemotingException) t); - } - - public boolean exceedMaxRetry() { - return this.retry.get() >= 3; - } - - public void incrRetry() { - this.retry.incrementAndGet(); - } - - public SendResult wrapSendResult(SendResult sendResult) { - if (sendResult == null) { - return null; - } - SendResultForLogicalQueue newSendResult = new SendResultForLogicalQueue(sendResult, this.writableQueueRouteData.getLogicalQueueIndex()); - long queueOffset = newSendResult.getQueueOffset(); - if (queueOffset >= 0) { - newSendResult.setQueueOffset(LogicalQueueSendContext.this.writableQueueRouteData.toLogicalQueueOffset(queueOffset)); - } - return newSendResult; - } - - public boolean handleRemotingException(RemotingException e) { - if (e instanceof RemotingTooMuchRequestException) { - return false; - } - DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId())); - this.logicalQueuesInfo = DefaultMQProducerImpl.this.getTopicPublishInfoTable().get(mq.getTopic()).getTopicRouteData().getLogicalQueuesInfo(); - LogicalQueueRouteData writableQueueRouteData; - try { - writableQueueRouteData = this.getWritableQueueRouteData(); - } catch (MQClientException ce) { - log.warn("getWritableQueueRouteData exception: {}", this.logicalQueuesInfo.get(mq.getQueueId()), ce); - return false; - } - if (Objects.equal(this.writableQueueRouteData.getMessageQueue(), writableQueueRouteData.getMessageQueue()) && writableQueueRouteData.isWritable()) { - // still same MessageQueue and still writable, no need to retry - return false; - } - return true; - } - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java b/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java deleted file mode 100644 index 09cd469d91..0000000000 --- a/client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.producer; - -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageQueue; - -public class SendResultForLogicalQueue extends SendResult { - private final String origBrokerName; - private final int origQueueId; - - public SendResultForLogicalQueue(SendResult sendResult, int logicalQueueIdx) { - super(sendResult.getSendStatus(), sendResult.getMsgId(), sendResult.getOffsetMsgId(), new MessageQueue(sendResult.getMessageQueue().getTopic(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx), sendResult.getQueueOffset()); - this.origBrokerName = sendResult.getMessageQueue().getBrokerName(); - this.origQueueId = sendResult.getMessageQueue().getQueueId(); - } - - public String getOrigBrokerName() { - return origBrokerName; - } - - public int getOrigQueueId() { - return origQueueId; - } - - @Override public String toString() { - return "SendResultForLogicalQueue{" + - "origBrokerName='" + origBrokerName + '\'' + - ", origQueueId=" + origQueueId + - "} " + super.toString(); - } -} diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java deleted file mode 100644 index 15ec564266..0000000000 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.consumer; - -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.SettableFuture; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.rocketmq.client.ClientConfig; -import org.apache.rocketmq.client.exception.MQRedirectException; -import org.apache.rocketmq.client.impl.CommunicationMode; -import org.apache.rocketmq.client.impl.FindBrokerResult; -import org.apache.rocketmq.client.impl.MQClientAPIImpl; -import org.apache.rocketmq.client.impl.MQClientManager; -import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; -import org.apache.rocketmq.client.impl.consumer.PullResultExt; -import org.apache.rocketmq.client.impl.factory.MQClientInstance; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.assertj.core.util.Lists; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class DefaultMQPullConsumerLogicalQueueTest { - private MQClientInstance mQClientFactory; - @Mock - private MQClientAPIImpl mQClientAPIImpl; - private DefaultMQPullConsumer pullConsumer; - private String topic; - private static final String cluster = "DefaultCluster"; - private static final String broker1Name = "BrokerA"; - private static final String broker1Addr = "127.0.0.2:10911"; - private static final String broker2Name = "BrokerB"; - private static final String broker2Addr = "127.0.0.3:10911"; - - @Before - public void init() throws Exception { - topic = "FooBar" + System.nanoTime(); - - mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig())); - - FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); - - pullConsumer = new DefaultMQPullConsumer("FooBarGroup" + System.nanoTime()); - pullConsumer.setNamesrvAddr("127.0.0.1:9876"); - pullConsumer.start(); - - PullAPIWrapper pullAPIWrapper = pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper(); - FieldUtils.writeDeclaredField(pullAPIWrapper, "mQClientFactory", mQClientFactory, true); - - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRouteData()); - - doReturn(new FindBrokerResult(broker1Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker1Name), anyLong(), anyBoolean()); - doReturn(new FindBrokerResult(broker2Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker2Name), anyLong(), anyBoolean()); - } - - @After - public void terminate() { - pullConsumer.shutdown(); - } - - @Test - public void testStart_OffsetShouldNotNUllAfterStart() { - Assert.assertNotNull(pullConsumer.getOffsetStore()); - } - - @Test - public void testPullMessage_Success() throws Exception { - doAnswer(new Answer() { - @Override public PullResultExt answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); - } - }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); - - MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); - PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); - assertThat(pullResult).isNotNull(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); - assertThat(pullResult.getMinOffset()).isEqualTo(123); - assertThat(pullResult.getMaxOffset()).isEqualTo(2048); - assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); - } - - @Test - public void testPullMessage_NotFound() throws Exception { - doAnswer(new Answer() { - @Override public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.NO_NEW_MSG, new ArrayList()); - } - }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); - - MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); - PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); - } - - @Test - public void testPullMessageAsync_Success() throws Exception { - doAnswer(new Answer() { - @Override public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - PullResult pullResult = DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); - - PullCallback pullCallback = mock.getArgument(4); - pullCallback.onSuccess(pullResult); - return null; - } - }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.ASYNC), any(PullCallback.class)); - - final SettableFuture future = SettableFuture.create(); - MessageQueue messageQueue = new MessageQueue(topic, broker1Name, 0); - pullConsumer.pull(messageQueue, "*", 1024, 3, new PullCallback() { - @Override - public void onSuccess(PullResult pullResult) { - future.set(pullResult); - } - - @Override - public void onException(Throwable e) { - future.setException(e); - } - }); - PullResult pullResult = future.get(3, TimeUnit.SECONDS); - assertThat(pullResult).isNotNull(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); - assertThat(pullResult.getMinOffset()).isEqualTo(123); - assertThat(pullResult.getMaxOffset()).isEqualTo(2048); - assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); - } - - @Test - public void testPullMessageSync_Redirect() throws Exception { - doAnswer(new Answer() { - @Override public PullResult answer(InvocationOnMock mock) throws Throwable { - throw new MQRedirectException(JSON.toJSONBytes(ImmutableList.of( - new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr), - new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr) - ))); - } - }).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); - doAnswer(new Answer() { - @Override public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt())); - } - }).when(mQClientAPIImpl).pullMessage(eq(broker2Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull()); - - MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); - PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3); - assertThat(pullResult).isNotNull(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1); - assertThat(pullResult.getMinOffset()).isEqualTo(123 + 10); - assertThat(pullResult.getMaxOffset()).isEqualTo(2048 + 10); - assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList()); - } - - private TopicRouteData createTopicRouteData() { - TopicRouteData topicRouteData = new TopicRouteData(); - - topicRouteData.setFilterServerTable(new HashMap>()); - topicRouteData.setBrokerDatas(ImmutableList.of( - new BrokerData(cluster, broker1Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))), - new BrokerData(cluster, broker2Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr))) - )); - - List queueDataList = new ArrayList(); - QueueData queueData; - queueData = new QueueData(); - queueData.setBrokerName(broker1Name); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSysFlag(0); - queueDataList.add(queueData); - queueData = new QueueData(); - queueData.setBrokerName(broker2Name); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSysFlag(0); - queueDataList.add(queueData); - topicRouteData.setQueueDatas(queueDataList); - - LogicalQueuesInfo info = new LogicalQueuesInfo(); - info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr))); - topicRouteData.setLogicalQueuesInfo(info); - return topicRouteData; - } - - private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, - List messageExtList) throws Exception { - return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, new byte[] {}); - } -} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java index f762910a40..73cfefb47a 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java @@ -20,10 +20,12 @@ import java.util.Collections; import java.util.HashSet; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.OffsetNotFoundException; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientAPIImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -60,6 +62,7 @@ public class RemoteBrokerOffsetStoreTest { when(mQClientFactory.getClientId()).thenReturn(clientId); when(mQClientFactory.findBrokerAddressInAdmin(brokerName)).thenReturn(new FindBrokerResult("127.0.0.1", false)); when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPI); + when(mQClientFactory.getBrokerNameFromMessageQueue(any())).thenReturn(brokerName); } @Test @@ -84,10 +87,15 @@ public class RemoteBrokerOffsetStoreTest { offsetStore.updateOffset(messageQueue, 1024, false); - doThrow(new MQBrokerException(-1, "", null)) + doThrow(new OffsetNotFoundException(ResponseCode.PULL_NOT_FOUND, "", null)) .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong()); assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1); + + doThrow(new MQBrokerException(-1, "", null)) + .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong()); + assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2); + doThrow(new RemotingException("", null)) .when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong()); assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java deleted file mode 100644 index 12d5cbaab8..0000000000 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.producer; - -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.SettableFuture; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.rocketmq.client.ClientConfig; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.exception.MQRedirectException; -import org.apache.rocketmq.client.hook.SendMessageContext; -import org.apache.rocketmq.client.impl.CommunicationMode; -import org.apache.rocketmq.client.impl.MQClientAPIImpl; -import org.apache.rocketmq.client.impl.MQClientManager; -import org.apache.rocketmq.client.impl.factory.MQClientInstance; -import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; -import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.assertj.core.api.ThrowableAssert; -import org.assertj.core.util.Lists; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -@RunWith(MockitoJUnitRunner.class) -public class DefaultMQProducerLogicalQueueTest { - private MQClientInstance mQClientFactory; - @Mock - private MQClientAPIImpl mQClientAPIImpl; - - private DefaultMQProducer producer; - private Message message; - private String topic; - - private MessageQueue messageQueue; - - private static final String cluster = "DefaultCluster"; - private static final String broker1Name = "broker1"; - private static final String broker2Name = "broker2"; - private static final String broker1Addr = "127.0.0.2:10911"; - private static final String broker2Addr = "127.0.0.3:10911"; - - @Before - public void init() throws Exception { - topic = "Foobar" + System.nanoTime(); - messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0); - - ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); - for (MQClientInstance instance : factoryTable.values()) { - instance.shutdown(); - } - factoryTable.clear(); - - mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig())); - factoryTable.put(new ClientConfig().buildMQClientId(), mQClientFactory); - - String producerGroupTemp = "FooBar_PID" + System.nanoTime(); - producer = new DefaultMQProducer(producerGroupTemp); - producer.setNamesrvAddr("127.0.0.1:9876"); - producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); - message = new Message(topic, new byte[] {'a'}); - - mQClientFactory.registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl()); - - producer.start(); - - FieldUtils.writeDeclaredField(producer.getDefaultMQProducerImpl(), "mQClientFactory", mQClientFactory, true); - FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); - - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - any(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenAnswer(new Answer() { - @Override public SendResult answer(InvocationOnMock invocation) throws Throwable { - SendCallback sendCallback = invocation.getArgument(6); - sendCallback.onSuccess(DefaultMQProducerLogicalQueueTest.this.createSendResult(SendStatus.SEND_OK)); - return null; - } - }); - } - - @After - public void terminate() { - producer.shutdown(); - } - - @Test - public void testSendMessageSync_Success() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); - SendResult sendResult = producer.send(message, messageQueue); - - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); - assertThat(sendResult.getQueueOffset()).isEqualTo(456L); - } - - @Test - public void testSendMessageSync_Redirect() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); - - when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenThrow(new MQRedirectException(null)); - - assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { - @Override public void call() throws Throwable { - producer.send(message, messageQueue); - } - }).isInstanceOf(MQBrokerException.class).hasMessageContaining("redirect"); - - when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenThrow(new MQRedirectException(JSON.toJSONBytes(ImmutableList.of( - new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr), - new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr))))); - when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); - - SendResult sendResult = producer.send(message, messageQueue); - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); - assertThat(sendResult.getQueueOffset()).isEqualTo(466L); - } - - @Test - public void testSendMessageSync_RemotingException() throws Exception { - TopicRouteData topicRouteData = createTopicRoute(); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(topicRouteData); - - when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenThrow(new RemotingConnectException(broker1Addr)); - SendResult returnSendResult = createSendResult(SendStatus.SEND_OK); - when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), - (SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(returnSendResult); - - assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { - @Override public void call() throws Throwable { - producer.send(message, messageQueue); - } - }).isInstanceOf(RemotingConnectException.class).hasMessageContaining(broker1Addr); - - topicRouteData.getLogicalQueuesInfo().get(0).add(new LogicalQueueRouteData(0, -1, new MessageQueue(topic, broker2Name, 1), MessageQueueRouteState.WriteOnly, 0, -1, -1, -1, broker2Addr)); - - SendResult sendResult = producer.send(message, messageQueue); - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); - assertThat(sendResult.getQueueOffset()).isEqualTo(-1L); - } - - @Test - public void testSendMessageAsync_Success() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); - - final SettableFuture future = SettableFuture.create(); - producer.send(message, messageQueue, new SendCallback() { - @Override - public void onSuccess(SendResult sendResult) { - future.set(sendResult); - } - - @Override - public void onException(Throwable e) { - future.setException(e); - } - }); - - SendResult sendResult = future.get(3, TimeUnit.SECONDS); - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); - assertThat(sendResult.getQueueOffset()).isEqualTo(456L); - } - - @Test - public void testSendMessageAsync() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); - - final AtomicReference> future = new AtomicReference>(); - SendCallback sendCallback = new SendCallback() { - @Override - public void onSuccess(SendResult sendResult) { - future.get().set(sendResult); - } - - @Override - public void onException(Throwable e) { - future.get().setException(e); - } - }; - - Message message = new Message(); - message.setTopic("test"); - message.setBody("hello world".getBytes()); - future.set(SettableFuture.create()); - producer.send(new Message(), messageQueue, sendCallback); - assertThatThrownBy(new ThrowableAssert.ThrowingCallable() { - @Override public void call() throws Throwable { - future.get().get(3, TimeUnit.SECONDS); - } - }).hasCauseInstanceOf(MQClientException.class).hasMessageContaining("The specified topic is blank"); - - //this message is send success - message.setTopic(topic); - future.set(SettableFuture.create()); - producer.send(message, messageQueue, sendCallback, 1000); - future.get().get(3, TimeUnit.SECONDS); - } - - public TopicRouteData createTopicRoute() { - TopicRouteData topicRouteData = new TopicRouteData(); - - topicRouteData.setFilterServerTable(new HashMap>()); - topicRouteData.setBrokerDatas(ImmutableList.of( - new BrokerData(cluster, broker1Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))), - new BrokerData(cluster, broker2Name, new HashMap(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr))) - )); - - List queueDataList = new ArrayList(); - QueueData queueData; - queueData = new QueueData(); - queueData.setBrokerName(broker1Name); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSysFlag(0); - queueDataList.add(queueData); - queueData = new QueueData(); - queueData.setBrokerName(broker2Name); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSysFlag(0); - queueDataList.add(queueData); - topicRouteData.setQueueDatas(queueDataList); - - LogicalQueuesInfo info = new LogicalQueuesInfo(); - info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr))); - topicRouteData.setLogicalQueuesInfo(info); - return topicRouteData; - } - - private SendResult createSendResult(SendStatus sendStatus) { - SendResult sendResult = new SendResult(); - sendResult.setMsgId("123"); - sendResult.setOffsetMsgId("123"); - sendResult.setQueueOffset(456); - sendResult.setSendStatus(sendStatus); - sendResult.setRegionId("HZ"); - sendResult.setMessageQueue(new MessageQueue(topic, broker1Name, 0)); - return sendResult; - } -} diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index a8906b33d2..5f29fe113c 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -49,20 +48,21 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -153,7 +153,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageSync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendResult sendResult = producer.send(message); assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); @@ -163,7 +163,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageSync_WithBodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendResult sendResult = producer.send(bigMessage); assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); @@ -174,7 +174,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -197,7 +197,7 @@ public class DefaultMQProducerTest { final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(6); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -239,7 +239,7 @@ public class DefaultMQProducerTest { final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(4); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -260,7 +260,7 @@ public class DefaultMQProducerTest { } }; - List msgs = new ArrayList(); + List msgs = new ArrayList<>(); for (int i = 0; i < 5; i++) { Message message = new Message(); message.setTopic("test"); @@ -281,7 +281,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -300,7 +300,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageSync_SuccessWithHook() throws Throwable { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); final Throwable[] assertionErrors = new Throwable[1]; final CountDownLatch countDownLatch = new CountDownLatch(2); producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() { @@ -368,7 +368,7 @@ public class DefaultMQProducerTest { @Test public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); final AtomicBoolean finish = new AtomicBoolean(false); new Thread(new Runnable() { @Override public void run() { @@ -394,13 +394,13 @@ public class DefaultMQProducerTest { @Test(expected = RequestTimeoutException.class) public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); Message result = producer.request(message, 3 * 1000L); } @Test public void testAsyncRequest_OnSuccess() throws Exception { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); RequestCallback requestCallback = new RequestCallback() { @Override public void onSuccess(Message message) { diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java index 0a1f685026..5d64a93f03 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java @@ -20,11 +20,6 @@ package org.apache.rocketmq.client.trace; import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -52,14 +47,17 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -115,7 +113,7 @@ public class DefaultMQProducerWithOpenTracingTest { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message); assertThat(tracer.finishedSpans().size()).isEqualTo(1); MockSpan span = tracer.finishedSpans().get(0); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java index 64f63c59aa..234e32e680 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java @@ -17,13 +17,6 @@ package org.apache.rocketmq.client.trace; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -50,17 +43,18 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -128,7 +122,7 @@ public class DefaultMQProducerWithTraceTest { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); try { producer.send(message); @@ -140,7 +134,7 @@ public class DefaultMQProducerWithTraceTest { @Test public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); final CountDownLatch countDownLatch = new CountDownLatch(1); try { producer.send(message); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java index aca62544d8..dd6d1083ce 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java @@ -20,12 +20,6 @@ package org.apache.rocketmq.client.trace; import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -59,14 +53,18 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -133,7 +131,7 @@ public class TransactionMQProducerWithOpenTracingTest { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.sendMessageInTransaction(message, null); assertThat(tracer.finishedSpans().size()).isEqualTo(2); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java index b3a441433c..f838817bf9 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java @@ -17,13 +17,6 @@ package org.apache.rocketmq.client.trace; -import java.lang.reflect.Field; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -57,20 +50,19 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; import org.mockito.Mock; import org.mockito.Spy; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; + +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.when; @@ -135,7 +127,7 @@ public class TransactionMQProducerWithTraceTest { Field fieldHooks = DefaultMQProducerImpl.class.getDeclaredField("endTransactionHookList"); fieldHooks.setAccessible(true); - Listhooks = new ArrayList(); + Listhooks = new ArrayList<>(); hooks.add(endTransactionHook); fieldHooks.set(producer.getDefaultMQProducerImpl(), hooks); @@ -150,14 +142,12 @@ public class TransactionMQProducerWithTraceTest { @Test public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.>any())).thenReturn(createTopicRoute()); - final AtomicReference context = new AtomicReference(); - doAnswer(new Answer() { - @Override public Object answer(InvocationOnMock mock) throws Throwable { - context.set(mock.getArgument(0)); - return null; - } - }).when(endTransactionHook).endTransaction(ArgumentMatchers.any()); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + AtomicReference context = new AtomicReference<>(); + doAnswer(mock -> { + context.set(mock.getArgument(0)); + return null; + }).when(endTransactionHook).endTransaction(any()); producer.sendMessageInTransaction(message, null); EndTransactionContext ctx = context.get(); diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 8a7183f535..cc12077734 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -16,7 +16,12 @@ */ package org.apache.rocketmq.common; -import com.alibaba.fastjson.TypeReference; +import org.apache.rocketmq.common.annotation.ImportantField; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -27,7 +32,6 @@ import java.lang.annotation.Annotation; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.lang.reflect.Type; import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; @@ -40,12 +44,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.concurrent.atomic.AtomicLong; -import org.apache.rocketmq.common.annotation.ImportantField; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; public class MixAll { public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; @@ -87,9 +85,9 @@ public class MixAll { public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; public static final String REPLY_MESSAGE_FLAG = "reply"; private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); - public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME = "__logical_queue_broker__"; - public static final Type TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA = new TypeReference>() { - }.getType(); + public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = "__syslo__"; + public static final String METADATA_SCOPE_GLOBAL = "__global__"; + public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = "__syslo__none__"; public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); diff --git a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java index c082ba6616..ec4d54bc96 100644 --- a/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/TopicConfig.java @@ -47,6 +47,12 @@ public class TopicConfig { this.order = other.order; } + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums) { + this.topicName = topicName; + this.readQueueNums = readQueueNums; + this.writeQueueNums = writeQueueNums; + } + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) { this.topicName = topicName; this.readQueueNums = readQueueNums; diff --git a/common/src/main/java/org/apache/rocketmq/common/admin/TopicOffset.java b/common/src/main/java/org/apache/rocketmq/common/admin/TopicOffset.java index 7e66749183..8b52a88a10 100644 --- a/common/src/main/java/org/apache/rocketmq/common/admin/TopicOffset.java +++ b/common/src/main/java/org/apache/rocketmq/common/admin/TopicOffset.java @@ -44,4 +44,13 @@ public class TopicOffset { public void setLastUpdateTimestamp(long lastUpdateTimestamp) { this.lastUpdateTimestamp = lastUpdateTimestamp; } + + @Override + public String toString() { + return "TopicOffset{" + + "minOffset=" + minOffset + + ", maxOffset=" + maxOffset + + ", lastUpdateTimestamp=" + lastUpdateTimestamp + + '}'; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 04f126b22f..534a2d25ec 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -200,13 +200,6 @@ public class RequestCode { public static final int QUERY_ASSIGNMENT = 400; public static final int SET_MESSAGE_REQUEST_MODE = 401; - public static final int UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING = 411; - public static final int DELETE_TOPIC_LOGICAL_QUEUE_MAPPING = 422; - public static final int QUERY_TOPIC_LOGICAL_QUEUE_MAPPING = 413; - public static final int SEAL_TOPIC_LOGICAL_QUEUE = 414; - public static final int REUSE_TOPIC_LOGICAL_QUEUE = 415; - public static final int CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE = 416; - public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE = 417; - public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT = 418; - public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY = 419; + public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513; + } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java index 42b9c4fa0f..3f691b5414 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java @@ -84,4 +84,13 @@ public class ResponseCode extends RemotingSysResponseCode { public static final int POLLING_FULL = 209; public static final int POLLING_TIMEOUT = 210; + + public static final int NOT_LEADER_FOR_QUEUE = 501; + + public static final int RPC_UNKNOWN = -1000; + public static final int RPC_ADDR_IS_NULL = -1002; + public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; + public static final int RPC_TIME_OUT = -1006; + + } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java deleted file mode 100644 index e446d9ba4c..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.body; - -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; - -public class CreateMessageQueueForLogicalQueueRequestBody extends RemotingSerializable { - private String topic; - private int logicalQueueIndex; - private MessageQueueRouteState messageQueueStatus; - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public int getLogicalQueueIndex() { - return logicalQueueIndex; - } - - public void setLogicalQueueIndex(int logicalQueueIndex) { - this.logicalQueueIndex = logicalQueueIndex; - } - - public MessageQueueRouteState getMessageQueueStatus() { - return messageQueueStatus; - } - - public void setMessageQueueStatus(MessageQueueRouteState messageQueueStatuses) { - this.messageQueueStatus = messageQueueStatuses; - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java deleted file mode 100644 index 6eb06a5d97..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.body; - -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; - -public class MigrateLogicalQueueBody extends RemotingSerializable { - private LogicalQueueRouteData fromQueueRouteData; - private LogicalQueueRouteData toQueueRouteData; - - public LogicalQueueRouteData getFromQueueRouteData() { - return fromQueueRouteData; - } - - public void setFromQueueRouteData( - LogicalQueueRouteData fromQueueRouteData) { - this.fromQueueRouteData = fromQueueRouteData; - } - - public LogicalQueueRouteData getToQueueRouteData() { - return toQueueRouteData; - } - - public void setToQueueRouteData(LogicalQueueRouteData toQueueRouteData) { - this.toQueueRouteData = toQueueRouteData; - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java index 4065c08792..1921727aa0 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java @@ -33,6 +33,7 @@ import java.util.zip.InflaterInputStream; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -41,7 +42,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class RegisterBrokerBody extends RemotingSerializable { private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); - private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + private TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper(); private List filterServerList = new ArrayList(); public byte[] encode(boolean compress) { @@ -82,6 +83,20 @@ public class RegisterBrokerBody extends RemotingSerializable { // write filter server list json outputStream.write(buffer); + //write the topic queue mapping + Map topicQueueMappingInfoMap = topicConfigSerializeWrapper.getTopicQueueMappingInfoMap(); + if (topicQueueMappingInfoMap == null) { + //as the place holder + topicQueueMappingInfoMap = new ConcurrentHashMap(); + } + outputStream.write(convertIntToByteArray(topicQueueMappingInfoMap.size())); + for (TopicQueueMappingInfo info: topicQueueMappingInfoMap.values()) { + buffer = JSON.toJSONString(info).getBytes(MixAll.DEFAULT_CHARSET); + outputStream.write(convertIntToByteArray(buffer.length)); + // write filter server list json + outputStream.write(buffer); + } + outputStream.finish(); long interval = System.currentTimeMillis() - start; if (interval > 50) { @@ -134,6 +149,17 @@ public class RegisterBrokerBody extends RemotingSerializable { } registerBrokerBody.setFilterServerList(filterServerList); + + int topicQueueMappingNum = readInt(inflaterInputStream); + Map topicQueueMappingInfoMap = new ConcurrentHashMap(); + for (int i = 0; i < topicQueueMappingNum; i++) { + int mappingJsonLen = readInt(inflaterInputStream); + byte[] buffer = readBytes(inflaterInputStream, mappingJsonLen); + TopicQueueMappingInfo info = TopicQueueMappingInfo.decode(buffer, TopicQueueMappingInfo.class); + topicQueueMappingInfoMap.put(info.getTopic(), info); + } + registerBrokerBody.getTopicConfigSerializeWrapper().setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); + long interval = System.currentTimeMillis() - start; if (interval > 50) { LOGGER.info("Decompressing takes {}ms", interval); @@ -167,11 +193,11 @@ public class RegisterBrokerBody extends RemotingSerializable { return byteBuffer.getInt(); } - public TopicConfigSerializeWrapper getTopicConfigSerializeWrapper() { + public TopicConfigAndMappingSerializeWrapper getTopicConfigSerializeWrapper() { return topicConfigSerializeWrapper; } - public void setTopicConfigSerializeWrapper(TopicConfigSerializeWrapper topicConfigSerializeWrapper) { + public void setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper) { this.topicConfigSerializeWrapper = topicConfigSerializeWrapper; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java deleted file mode 100644 index 22ab452292..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.body; - -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; - -public class ReuseTopicLogicalQueueRequestBody extends RemotingSerializable { - private String topic; - private int queueId; - private int logicalQueueIndex; - private MessageQueueRouteState messageQueueRouteState; - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public int getQueueId() { - return queueId; - } - - public void setQueueId(int queueId) { - this.queueId = queueId; - } - - public int getLogicalQueueIndex() { - return logicalQueueIndex; - } - - public void setLogicalQueueIndex(int logicalQueueIndex) { - this.logicalQueueIndex = logicalQueueIndex; - } - - public void setMessageQueueRouteState(MessageQueueRouteState messageQueueRouteState) { - this.messageQueueRouteState = messageQueueRouteState; - } - - public MessageQueueRouteState getMessageQueueRouteState() { - return messageQueueRouteState; - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigAndMappingSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigAndMappingSerializeWrapper.java new file mode 100644 index 0000000000..830a8a43d9 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigAndMappingSerializeWrapper.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.protocol.body; + +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class TopicConfigAndMappingSerializeWrapper extends TopicConfigSerializeWrapper { + private Map topicQueueMappingInfoMap = new ConcurrentHashMap(); + + private Map topicQueueMappingDetailMap = new ConcurrentHashMap(); + + private DataVersion mappingDataVersion = new DataVersion(); + + + public Map getTopicQueueMappingInfoMap() { + return topicQueueMappingInfoMap; + } + + public void setTopicQueueMappingInfoMap(Map topicQueueMappingInfoMap) { + this.topicQueueMappingInfoMap = topicQueueMappingInfoMap; + } + + public Map getTopicQueueMappingDetailMap() { + return topicQueueMappingDetailMap; + } + + public void setTopicQueueMappingDetailMap(Map topicQueueMappingDetailMap) { + this.topicQueueMappingDetailMap = topicQueueMappingDetailMap; + } + + public DataVersion getMappingDataVersion() { + return mappingDataVersion; + } + + public void setMappingDataVersion(DataVersion mappingDataVersion) { + this.mappingDataVersion = mappingDataVersion; + } + + public static TopicConfigAndMappingSerializeWrapper from(TopicConfigSerializeWrapper wrapper) { + if (wrapper instanceof TopicConfigAndMappingSerializeWrapper) { + return (TopicConfigAndMappingSerializeWrapper) wrapper; + } + TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper = new TopicConfigAndMappingSerializeWrapper(); + mappingSerializeWrapper.setDataVersion(wrapper.getDataVersion()); + mappingSerializeWrapper.setTopicConfigTable(wrapper.getTopicConfigTable()); + return mappingSerializeWrapper; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java index 13896636fc..ce123021d4 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java @@ -17,18 +17,15 @@ package org.apache.rocketmq.common.protocol.body; -import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class TopicConfigSerializeWrapper extends RemotingSerializable { private ConcurrentMap topicConfigTable = new ConcurrentHashMap(); - private Map logicalQueuesInfoMap; private DataVersion dataVersion = new DataVersion(); public ConcurrentMap getTopicConfigTable() { @@ -46,12 +43,4 @@ public class TopicConfigSerializeWrapper extends RemotingSerializable { public void setDataVersion(DataVersion dataVersion) { this.dataVersion = dataVersion; } - - public Map getLogicalQueuesInfoMap() { - return logicalQueuesInfoMap; - } - - public void setLogicalQueuesInfoMap(Map logicalQueuesInfoMap) { - this.logicalQueuesInfoMap = logicalQueuesInfoMap; - } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicQueueMappingSerializeWrapper.java similarity index 53% rename from common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java rename to common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicQueueMappingSerializeWrapper.java index 67c6fd2a1a..799f81ae23 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicQueueMappingSerializeWrapper.java @@ -14,36 +14,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.rocketmq.common.protocol.body; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; -public class UpdateTopicLogicalQueueMappingRequestBody extends RemotingSerializable { - private String topic; - private int queueId; - private int logicalQueueIdx; +import java.util.Map; - public int getLogicalQueueIdx() { - return logicalQueueIdx; +public class TopicQueueMappingSerializeWrapper extends RemotingSerializable { + private Map topicQueueMappingInfoMap; + private DataVersion dataVersion = new DataVersion(); + + public Map getTopicQueueMappingInfoMap() { + return topicQueueMappingInfoMap; } - public void setLogicalQueueIdx(int logicalQueueIdx) { - this.logicalQueueIdx = logicalQueueIdx; + public void setTopicQueueMappingInfoMap(Map topicQueueMappingInfoMap) { + this.topicQueueMappingInfoMap = topicQueueMappingInfoMap; } - public String getTopic() { - return topic; + public DataVersion getDataVersion() { + return dataVersion; } - public void setTopic(String topic) { - this.topic = topic; - } - - public int getQueueId() { - return queueId; - } - - public void setQueueId(int queueId) { - this.queueId = queueId; + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java index 8894d0b938..290ec4c621 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java @@ -23,6 +23,7 @@ package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; public class CreateTopicRequestHeader implements CommandCustomHeader { @@ -42,6 +43,9 @@ public class CreateTopicRequestHeader implements CommandCustomHeader { @CFNotNull private Boolean order = false; + @CFNullable + private Boolean force = false; + @Override public void checkFields() throws RemotingCommandException { try { @@ -118,4 +122,12 @@ public class CreateTopicRequestHeader implements CommandCustomHeader { public void setOrder(Boolean order) { this.order = order; } + + public Boolean getForce() { + return force; + } + + public void setForce(Boolean force) { + this.force = force; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java index c64381fb78..acf4497146 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java @@ -20,11 +20,11 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader { +public class GetEarliestMsgStoretimeRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String topic; @CFNotNull @@ -34,18 +34,22 @@ public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader public void checkFields() throws RemotingCommandException { } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java index 6963195f91..e961af95e1 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -20,51 +20,38 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class GetMaxOffsetRequestHeader implements CommandCustomHeader { +public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String topic; @CFNotNull private Integer queueId; - private boolean committed; - private boolean logicalQueue; @Override public void checkFields() throws RemotingCommandException { } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } - public void setCommitted(boolean committed) { - this.committed = committed; - } - - public boolean isCommitted() { - return committed; - } - - public void setLogicalQueue(boolean logicalQueue) { - this.logicalQueue = logicalQueue; - } - - public boolean getLogicalQueue() { - return logicalQueue; - } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java index 6fb8ed40c4..70189b74b5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java @@ -20,11 +20,11 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class GetMinOffsetRequestHeader implements CommandCustomHeader { +public class GetMinOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String topic; @CFNotNull @@ -34,18 +34,22 @@ public class GetMinOffsetRequestHeader implements CommandCustomHeader { public void checkFields() throws RemotingCommandException { } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java index ea9d17c3c7..bc7586a627 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java @@ -17,11 +17,11 @@ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class GetTopicConfigRequestHeader implements CommandCustomHeader { +public class GetTopicConfigRequestHeader extends TopicRequestHeader { @Override public void checkFields() throws RemotingCommandException { } @@ -29,6 +29,7 @@ public class GetTopicConfigRequestHeader implements CommandCustomHeader { @CFNotNull private String topic; + /** * @return the topic */ diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java index c4cf4decdc..f98e150b12 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java @@ -17,11 +17,11 @@ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class GetTopicStatsInfoRequestHeader implements CommandCustomHeader { +public class GetTopicStatsInfoRequestHeader extends TopicRequestHeader { @CFNotNull private String topic; diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java index 106e89e511..e15170f084 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java @@ -20,12 +20,12 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class PullMessageRequestHeader implements CommandCustomHeader { +public class PullMessageRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String consumerGroup; @CFNotNull @@ -60,18 +60,22 @@ public class PullMessageRequestHeader implements CommandCustomHeader { this.consumerGroup = consumerGroup; } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java index 0112f7da8d..88af984df8 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java @@ -22,6 +22,7 @@ package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; public class PullMessageResponseHeader implements CommandCustomHeader { @@ -33,6 +34,8 @@ public class PullMessageResponseHeader implements CommandCustomHeader { private Long minOffset; @CFNotNull private Long maxOffset; + @CFNullable + private Long offsetDelta; @Override public void checkFields() throws RemotingCommandException { @@ -69,4 +72,12 @@ public class PullMessageResponseHeader implements CommandCustomHeader { public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) { this.suggestWhichBrokerId = suggestWhichBrokerId; } + + public Long getOffsetDelta() { + return offsetDelta; + } + + public void setOffsetDelta(Long offsetDelta) { + this.offsetDelta = offsetDelta; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java index 3b7f627c35..ebcbe0db2a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java @@ -20,11 +20,11 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader { +public class QueryConsumerOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String consumerGroup; @CFNotNull @@ -32,6 +32,8 @@ public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader { @CFNotNull private Integer queueId; + private Boolean setZeroIfNotFound; + @Override public void checkFields() throws RemotingCommandException { } @@ -44,19 +46,31 @@ public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader { this.consumerGroup = consumerGroup; } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } + + public Boolean getSetZeroIfNotFound() { + return setZeroIfNotFound; + } + + public void setSetZeroIfNotFound(Boolean setZeroIfNotFound) { + this.setZeroIfNotFound = setZeroIfNotFound; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java index 5ea2e24bfc..c8291d2665 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java @@ -20,11 +20,11 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class SearchOffsetRequestHeader implements CommandCustomHeader { +public class SearchOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String topic; @CFNotNull @@ -37,18 +37,22 @@ public class SearchOffsetRequestHeader implements CommandCustomHeader { } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java index 2df31e6bb2..808bc2d3d5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java @@ -20,12 +20,12 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class SendMessageRequestHeader implements CommandCustomHeader { +public class SendMessageRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String producerGroup; @CFNotNull @@ -64,10 +64,12 @@ public class SendMessageRequestHeader implements CommandCustomHeader { this.producerGroup = producerGroup; } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } @@ -88,10 +90,12 @@ public class SendMessageRequestHeader implements CommandCustomHeader { this.defaultTopicQueueNums = defaultTopicQueueNums; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java index 3f44db645c..11eccd5c1e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java @@ -20,11 +20,11 @@ */ package org.apache.rocketmq.common.protocol.header; -import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader { +public class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader { @CFNotNull private String consumerGroup; @CFNotNull @@ -46,18 +46,22 @@ public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader { this.consumerGroup = consumerGroup; } + @Override public String getTopic() { return topic; } + @Override public void setTopic(String topic) { this.topic = topic; } + @Override public Integer getQueueId() { return queueId; } + @Override public void setQueueId(Integer queueId) { this.queueId = queueId; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java index ad776c855b..a2806e6289 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java @@ -20,7 +20,6 @@ */ package org.apache.rocketmq.common.protocol.header.namesrv; -import java.util.Set; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -29,9 +28,6 @@ public class GetRouteInfoRequestHeader implements CommandCustomHeader { @CFNotNull private String topic; - private int sysFlag; - private Set logicalQueueIdsFilter; - @Override public void checkFields() throws RemotingCommandException { } @@ -43,20 +39,4 @@ public class GetRouteInfoRequestHeader implements CommandCustomHeader { public void setTopic(String topic) { this.topic = topic; } - - public int getSysFlag() { - return sysFlag; - } - - public void setSysFlag(int sysFlag) { - this.sysFlag = sysFlag; - } - - public void setLogicalQueueIdsFilter(Set filter) { - this.logicalQueueIdsFilter = filter; - } - - public Set getLogicalQueueIdsFilter() { - return logicalQueueIdsFilter; - } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java deleted file mode 100644 index aed4d5d669..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java +++ /dev/null @@ -1,309 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.route; - -import com.alibaba.fastjson.annotation.JSONField; -import com.google.common.base.Objects; -import com.google.common.base.Predicate; -import com.google.common.collect.Lists; -import java.util.List; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; - -/** - * logical queue offset -> message queue offset mapping - */ -public class LogicalQueueRouteData implements Comparable { - private volatile int logicalQueueIndex = -1; /* -1 means not set */ - private volatile long logicalQueueDelta = -1; /* inclusive, -1 means not set, occurred in writeOnly state */ - - private MessageQueue messageQueue; - - private volatile MessageQueueRouteState state = MessageQueueRouteState.Normal; - - private volatile long offsetDelta = 0; // valid when Normal/WriteOnly/ReadOnly - private volatile long offsetMax = -1; // exclusive, valid when ReadOnly - - private volatile long firstMsgTimeMillis = -1; // valid when ReadOnly - private volatile long lastMsgTimeMillis = -1; // valid when ReadOnly - - private String brokerAddr; /* not always set, only used by high availability forward */ - - public LogicalQueueRouteData() { - } - - public LogicalQueueRouteData(int logicalQueueIndex, long logicalQueueDelta, - MessageQueue messageQueue, MessageQueueRouteState state, long offsetDelta, long offsetMax, - long firstMsgTimeMillis, - long lastMsgTimeMillis, String brokerAddr) { - this.logicalQueueIndex = logicalQueueIndex; - this.logicalQueueDelta = logicalQueueDelta; - this.messageQueue = messageQueue; - this.state = state; - this.offsetDelta = offsetDelta; - this.offsetMax = offsetMax; - this.firstMsgTimeMillis = firstMsgTimeMillis; - this.lastMsgTimeMillis = lastMsgTimeMillis; - this.brokerAddr = brokerAddr; - } - - public LogicalQueueRouteData(LogicalQueueRouteData queueRouteData) { - copyFrom(queueRouteData); - } - - public int getLogicalQueueIndex() { - return logicalQueueIndex; - } - - public void setLogicalQueueIndex(int logicalQueueIndex) { - this.logicalQueueIndex = logicalQueueIndex; - } - - public long getLogicalQueueDelta() { - return logicalQueueDelta; - } - - public void setLogicalQueueDelta(long logicalQueueDelta) { - this.logicalQueueDelta = logicalQueueDelta; - } - - public MessageQueue getMessageQueue() { - return messageQueue; - } - - public void setMessageQueue(MessageQueue messageQueue) { - this.messageQueue = messageQueue; - } - - public MessageQueueRouteState getState() { - return state; - } - - @JSONField(serialize = false) - public int getStateOrdinal() { - return state.ordinal(); - } - - public void setState(MessageQueueRouteState state) { - this.state = state; - } - - public String getBrokerAddr() { - return brokerAddr; - } - - public void setBrokerAddr(String brokerAddr) { - this.brokerAddr = brokerAddr; - } - - public long getOffsetDelta() { - return offsetDelta; - } - - public void setOffsetDelta(long offsetDelta) { - this.offsetDelta = offsetDelta; - } - - public long getOffsetMax() { - return offsetMax; - } - - public void setOffsetMax(long offsetMax) { - this.offsetMax = offsetMax; - } - - public long getFirstMsgTimeMillis() { - return firstMsgTimeMillis; - } - - public void setFirstMsgTimeMillis(long firstMsgTimeMillis) { - this.firstMsgTimeMillis = firstMsgTimeMillis; - } - - public long getLastMsgTimeMillis() { - return lastMsgTimeMillis; - } - - public void setLastMsgTimeMillis(long lastMsgTimeMillis) { - this.lastMsgTimeMillis = lastMsgTimeMillis; - } - - @Override public String toString() { - return "LogicalQueueRouteData{" + - "logicalQueueIndex=" + logicalQueueIndex + - ", logicalQueueDelta=" + logicalQueueDelta + - ", messageQueue=" + messageQueue + - ", state=" + state + - ", offsetDelta=" + offsetDelta + - ", offsetMax=" + offsetMax + - ", firstMsgTimeMillis=" + firstMsgTimeMillis + - ", lastMsgTimeMillis=" + lastMsgTimeMillis + - ", brokerAddr='" + brokerAddr + '\'' + - '}'; - } - - public void copyFrom(LogicalQueueRouteData queueRouteData) { - this.logicalQueueIndex = queueRouteData.logicalQueueIndex; - this.logicalQueueDelta = queueRouteData.logicalQueueDelta; - this.messageQueue = new MessageQueue(queueRouteData.getMessageQueue()); - this.state = queueRouteData.state; - this.offsetDelta = queueRouteData.offsetDelta; - this.offsetMax = queueRouteData.offsetMax; - this.firstMsgTimeMillis = queueRouteData.firstMsgTimeMillis; - this.lastMsgTimeMillis = queueRouteData.lastMsgTimeMillis; - this.brokerAddr = queueRouteData.brokerAddr; - } - - public long toLogicalQueueOffset(long messageQueueOffset) { - return this.logicalQueueDelta < 0 ? -1 : messageQueueOffset - this.offsetDelta + this.logicalQueueDelta; - } - - public long toMessageQueueOffset(long logicalQueueOffset) { - return logicalQueueOffset - this.logicalQueueDelta + this.offsetDelta; - } - - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - LogicalQueueRouteData that = (LogicalQueueRouteData) o; - return logicalQueueIndex == that.logicalQueueIndex && logicalQueueDelta == that.logicalQueueDelta && offsetDelta == that.offsetDelta && offsetMax == that.offsetMax && firstMsgTimeMillis == that.firstMsgTimeMillis && lastMsgTimeMillis == that.lastMsgTimeMillis && Objects.equal(messageQueue, that.messageQueue) && state == that.state && Objects.equal(brokerAddr, that.brokerAddr); - } - - @Override public int hashCode() { - return Objects.hashCode(logicalQueueIndex, logicalQueueDelta, messageQueue, state, offsetDelta, offsetMax, firstMsgTimeMillis, lastMsgTimeMillis, brokerAddr); - } - - @JSONField(serialize = false) - public long getMessagesCount() { - return this.offsetDelta >= 0 && this.offsetMax >= 0 ? this.offsetMax - this.offsetDelta : 0L; - } - - @JSONField(serialize = false) - public boolean isWritable() { - return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.WriteOnly.equals(state); - } - - @JSONField(serialize = false) - public boolean isReadable() { - return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.ReadOnly.equals(state); - } - - @JSONField(serialize = false) - public boolean isExpired() { - return MessageQueueRouteState.Expired.equals(state); - } - - @JSONField(serialize = false) - public boolean isWriteOnly() { - return MessageQueueRouteState.WriteOnly.equals(state); - } - - @JSONField(serialize = false) - public int getQueueId() { - return messageQueue.getQueueId(); - } - - @JSONField(serialize = false) - public String getBrokerName() { - return messageQueue.getBrokerName(); - } - - @JSONField(serialize = false) - public String getTopic() { - return messageQueue.getTopic(); - } - - public boolean isSameTo(LogicalQueueRouteData o) { - if (o == null) { - return false; - } - return isSameTo(o.getMessageQueue(), o.offsetDelta); - } - - public boolean isSameTo(MessageQueue mq, long offsetDelta) { - return Objects.equal(this.messageQueue, mq) && this.offsetDelta == offsetDelta; - } - - /** - * First compare logicalQueueDelta, negative delta must be ordered in the last; then compare state's ordinal; then - * compare messageQueue, nulls first; then compare offsetDelta. - */ - @Override - public int compareTo(LogicalQueueRouteData o) { - long x = this.getLogicalQueueDelta(); - long y = o.getLogicalQueueDelta(); - int result; - { - if (x >= 0 && y >= 0) { - result = MixAll.compareLong(x, y); - } else if (x < 0 && y < 0) { - result = MixAll.compareLong(-x, -y); - } else if (x < 0) { - // o1 < 0 && o2 >= 0 - result = 1; - } else { - // o1 >= 0 && o2 < 0 - result = -1; - } - } - if (result == 0) { - result = MixAll.compareInteger(this.state.ordinal(), o.state.ordinal()); - } - if (result == 0) { - if (this.messageQueue == null) { - if (o.messageQueue != null) { - result = -1; - } - } else { - if (o.messageQueue != null) { - result = this.messageQueue.compareTo(o.messageQueue); - } else { - result = 1; - } - } - } - if (result == 0) { - result = MixAll.compareLong(this.offsetDelta, o.offsetDelta); - } - return result; - } - - public static final Predicate READABLE_PREDICT = new Predicate() { - @Override - public boolean apply(LogicalQueueRouteData input) { - return input != null && input.isReadable(); - } - }; - - public List filterMessages(List list) { - if (this.offsetMax < 0 || list == null || list.isEmpty()) { - return list; - } - List result = Lists.newArrayListWithExpectedSize(list.size()); - for (MessageExt m : list) { - if (m.getQueueOffset() >= this.offsetMax) { - break; - } else { - result.add(m); - } - } - return result; - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java deleted file mode 100644 index de62b435d6..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.route; - -import com.alibaba.fastjson.parser.ParserConfig; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; - -public class LogicalQueuesInfo extends HashMap> { - // TODO whether here needs more fine-grained locks like per logical queue lock? - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - public LogicalQueuesInfo() { - super(); - } - - public LogicalQueuesInfo(Map> m) { - super(m); - } - - public Lock readLock() { - return lock.readLock(); - } - - public Lock writeLock() { - return lock.writeLock(); - } - - public void updateLogicalQueueRouteDataList(int logicalQueueIdx, - List logicalQueueRouteDataList) { - this.writeLock().lock(); - try { - logicalQueueRouteDataList = Lists.newLinkedList(logicalQueueRouteDataList); - List queueRouteDataList = this.get(logicalQueueIdx); - for (LogicalQueueRouteData logicalQueueRouteData : queueRouteDataList) { - for (Iterator it = logicalQueueRouteDataList.iterator(); it.hasNext(); ) { - LogicalQueueRouteData newQueueRouteData = it.next(); - if (Objects.equal(newQueueRouteData.getMessageQueue(), logicalQueueRouteData.getMessageQueue()) && newQueueRouteData.getOffsetDelta() == logicalQueueRouteData.getOffsetDelta()) { - logicalQueueRouteData.copyFrom(newQueueRouteData); - it.remove(); - break; - } - } - if (logicalQueueRouteDataList.isEmpty()) { - break; - } - } - for (LogicalQueueRouteData queueRouteData : logicalQueueRouteDataList) { - int idx = Collections.binarySearch(queueRouteDataList, queueRouteData); - if (idx < 0) { - idx = -idx - 1; - } - queueRouteDataList.add(idx, queueRouteData); - } - } finally { - this.writeLock().unlock(); - } - } - - static { - // workaround https://github.com/alibaba/fastjson/issues/3730 - ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfo.class, GenericMapSuperclassDeserializer.INSTANCE); - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java deleted file mode 100644 index 73d6061eac..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.route; - -import com.alibaba.fastjson.parser.ParserConfig; -import com.google.common.base.Objects; -import com.google.common.collect.Lists; -import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Only used inside namesrv, between client and namesrv, to reduce cpu usage of namesrv - */ -public class LogicalQueuesInfoUnordered extends ConcurrentHashMap> { - static { - // workaround https://github.com/alibaba/fastjson/issues/3730 - ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoUnordered.class, GenericMapSuperclassDeserializer.INSTANCE); - } - - public LogicalQueuesInfoUnordered() { - super(); - } - - public LogicalQueuesInfoUnordered(int size) { - super(size); - } - - public LogicalQueuesInfo toLogicalQueuesInfoOrdered() { - LogicalQueuesInfo logicalQueuesInfoOrdered = new LogicalQueuesInfo(); - for (Map.Entry> entry : this.entrySet()) { - List list = Lists.newArrayListWithExpectedSize(entry.getValue().size()); - for (LogicalQueueRouteData d : entry.getValue().values()) { - list.add(new LogicalQueueRouteData(d)); - } - Collections.sort(list); - logicalQueuesInfoOrdered.put(entry.getKey(), list); - } - return logicalQueuesInfoOrdered; - } - - public static class Key { - private final String brokerName; - private final int queueId; - - private final long offsetDelta; - - private final int hash; - - public Key(String brokerName, int queueId, long offsetDelta) { - this.brokerName = brokerName; - this.queueId = queueId; - this.offsetDelta = offsetDelta; - - this.hash = Objects.hashCode(brokerName, queueId, this.offsetDelta); - } - - public String getBrokerName() { - return brokerName; - } - - public int getQueueId() { - return queueId; - } - - public long getOffsetDelta() { - return offsetDelta; - } - - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - Key id = (Key) o; - return queueId == id.queueId && offsetDelta == id.offsetDelta && Objects.equal(brokerName, id.brokerName); - } - - @Override public int hashCode() { - return hash; - } - - @Override public String toString() { - return "Key{" + - "brokerName='" + brokerName + '\'' + - ", queueId=" + queueId + - ", offsetDelta=" + offsetDelta + - '}'; - } - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java index 4470a2a516..3fe46309c0 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java @@ -23,6 +23,9 @@ package org.apache.rocketmq.common.protocol.route; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; + +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; public class TopicRouteData extends RemotingSerializable { @@ -30,7 +33,8 @@ public class TopicRouteData extends RemotingSerializable { private List queueDatas; private List brokerDatas; private HashMap/* Filter Server */> filterServerTable; - private LogicalQueuesInfo logicalQueuesInfo; + //It could be null or empty + private Map topicQueueMappingByBroker; public TopicRouteData() { } @@ -53,8 +57,8 @@ public class TopicRouteData extends RemotingSerializable { this.filterServerTable.putAll(topicRouteData.filterServerTable); } - if (topicRouteData.logicalQueuesInfo != null) { - this.logicalQueuesInfo = new LogicalQueuesInfo(topicRouteData.logicalQueuesInfo); + if (topicRouteData.topicQueueMappingByBroker != null) { + this.topicQueueMappingByBroker = new HashMap(topicRouteData.topicQueueMappingByBroker); } } @@ -90,12 +94,12 @@ public class TopicRouteData extends RemotingSerializable { this.orderTopicConf = orderTopicConf; } - public LogicalQueuesInfo getLogicalQueuesInfo() { - return logicalQueuesInfo; + public Map getTopicQueueMappingByBroker() { + return topicQueueMappingByBroker; } - public void setLogicalQueuesInfo(LogicalQueuesInfo logicalQueuesInfo) { - this.logicalQueuesInfo = logicalQueuesInfo; + public void setTopicQueueMappingByBroker(Map topicQueueMappingByBroker) { + this.topicQueueMappingByBroker = topicQueueMappingByBroker; } @Override @@ -106,7 +110,7 @@ public class TopicRouteData extends RemotingSerializable { result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode()); result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode()); result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode()); - result = prime * result + ((logicalQueuesInfo == null) ? 0 : logicalQueuesInfo.hashCode()); + result = prime * result + ((topicQueueMappingByBroker == null) ? 0 : topicQueueMappingByBroker.hashCode()); return result; } @@ -139,10 +143,10 @@ public class TopicRouteData extends RemotingSerializable { return false; } else if (!filterServerTable.equals(other.filterServerTable)) return false; - if (logicalQueuesInfo == null) { - if (other.logicalQueuesInfo != null) + if (topicQueueMappingByBroker == null) { + if (other.topicQueueMappingByBroker != null) return false; - } else if (!logicalQueuesInfo.equals(other.logicalQueuesInfo)) + } else if (!topicQueueMappingByBroker.equals(other.topicQueueMappingByBroker)) return false; return true; } @@ -150,6 +154,6 @@ public class TopicRouteData extends RemotingSerializable { @Override public String toString() { return "TopicRouteData [orderTopicConf=" + orderTopicConf + ", queueDatas=" + queueDatas - + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + ", logicalQueuesInfo=" + logicalQueuesInfo + "]"; + + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + ", topicQueueMappingInfoTable=" + topicQueueMappingByBroker + "]"; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java b/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java deleted file mode 100644 index e9fb84e975..0000000000 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.protocol.route; - -import com.google.common.base.Objects; - -public class TopicRouteDataNameSrv extends TopicRouteData { - private LogicalQueuesInfoUnordered logicalQueuesInfoUnordered; - - public TopicRouteDataNameSrv() { - } - - public LogicalQueuesInfoUnordered getLogicalQueuesInfoUnordered() { - return logicalQueuesInfoUnordered; - } - - public void setLogicalQueuesInfoUnordered( - LogicalQueuesInfoUnordered logicalQueuesInfoUnordered) { - this.logicalQueuesInfoUnordered = logicalQueuesInfoUnordered; - } - - @Override public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - if (!super.equals(o)) - return false; - TopicRouteDataNameSrv srv = (TopicRouteDataNameSrv) o; - return Objects.equal(logicalQueuesInfoUnordered, srv.logicalQueuesInfoUnordered); - } - - @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), logicalQueuesInfoUnordered); - } - - @Override public String toString() { - return "TopicRouteDataNameSrv{" + - "logicalQueuesInfoUnordered=" + logicalQueuesInfoUnordered + - "} " + super.toString(); - } - - public TopicRouteData toTopicRouteData() { - TopicRouteData topicRouteData = new TopicRouteData(this); - if (this.logicalQueuesInfoUnordered != null) { - topicRouteData.setLogicalQueuesInfo(this.logicalQueuesInfoUnordered.toLogicalQueuesInfoOrdered()); - } - return topicRouteData; - } -} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/ClientMetadata.java b/common/src/main/java/org/apache/rocketmq/common/rpc/ClientMetadata.java new file mode 100644 index 0000000000..28a5e64aad --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/ClientMetadata.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class ClientMetadata { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + + private final ConcurrentMap topicRouteTable = new ConcurrentHashMap(); + private final ConcurrentMap> topicEndPointsTable = new ConcurrentHashMap>(); + private final ConcurrentMap> brokerAddrTable = + new ConcurrentHashMap>(); + private final ConcurrentMap> brokerVersionTable = + new ConcurrentHashMap>(); + + public void freshTopicRoute(String topic, TopicRouteData topicRouteData) { + if (topic == null + || topicRouteData == null) { + return; + } + TopicRouteData old = this.topicRouteTable.get(topic); + if (!topicRouteDataIsChange(old, topicRouteData)) { + return ; + } + { + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs()); + } + } + { + ConcurrentMap mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData); + if (mqEndPoints != null + && !mqEndPoints.isEmpty()) { + topicEndPointsTable.put(topic, mqEndPoints); + } + } + } + + + public static boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) { + if (olddata == null || nowdata == null) + return true; + TopicRouteData old = new TopicRouteData(olddata); + TopicRouteData now = new TopicRouteData(nowdata); + Collections.sort(old.getQueueDatas()); + Collections.sort(old.getBrokerDatas()); + Collections.sort(now.getQueueDatas()); + Collections.sort(now.getBrokerDatas()); + return !old.equals(now); + + } + + public String getBrokerNameFromMessageQueue(final MessageQueue mq) { + if (topicEndPointsTable.get(mq.getTopic()) != null + && !topicEndPointsTable.get(mq.getTopic()).isEmpty()) { + return topicEndPointsTable.get(mq.getTopic()).get(mq); + } + return mq.getBrokerName(); + } + + public void refreshClusterInfo(ClusterInfo clusterInfo) { + if (clusterInfo == null + || clusterInfo.getBrokerAddrTable() == null) { + return; + } + for (Map.Entry entry : clusterInfo.getBrokerAddrTable().entrySet()) { + brokerAddrTable.put(entry.getKey(), entry.getValue().getBrokerAddrs()); + } + } + + public String findMasterBrokerAddr(String brokerName) { + if (!brokerAddrTable.containsKey(brokerName)) { + return null; + } + return brokerAddrTable.get(brokerName).get(MixAll.MASTER_ID); + } + + public ConcurrentMap> getBrokerAddrTable() { + return brokerAddrTable; + } + + public static ConcurrentMap topicRouteData2EndpointsForStaticTopic(final String topic, final TopicRouteData route) { + if (route.getTopicQueueMappingByBroker() == null + || route.getTopicQueueMappingByBroker().isEmpty()) { + return new ConcurrentHashMap(); + } + ConcurrentMap mqEndPointsOfBroker = new ConcurrentHashMap(); + + Map> mappingInfosByScope = new HashMap>(); + for (Map.Entry entry : route.getTopicQueueMappingByBroker().entrySet()) { + TopicQueueMappingInfo info = entry.getValue(); + String scope = info.getScope(); + if (scope != null) { + if (!mappingInfosByScope.containsKey(scope)) { + mappingInfosByScope.put(scope, new HashMap()); + } + mappingInfosByScope.get(scope).put(entry.getKey(), entry.getValue()); + } + } + + for (Map.Entry> mapEntry : mappingInfosByScope.entrySet()) { + String scope = mapEntry.getKey(); + Map topicQueueMappingInfoMap = mapEntry.getValue(); + ConcurrentMap mqEndPoints = new ConcurrentHashMap(); + List> mappingInfos = new ArrayList>(topicQueueMappingInfoMap.entrySet()); + Collections.sort(mappingInfos, new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return (int) (o2.getValue().getEpoch() - o1.getValue().getEpoch()); + } + }); + int maxTotalNums = 0; + long maxTotalNumOfEpoch = -1; + for (Map.Entry entry : mappingInfos) { + TopicQueueMappingInfo info = entry.getValue(); + if (info.getEpoch() >= maxTotalNumOfEpoch && info.getTotalQueues() > maxTotalNums) { + maxTotalNums = info.getTotalQueues(); + } + for (Map.Entry idEntry : entry.getValue().getCurrIdMap().entrySet()) { + int globalId = idEntry.getKey(); + MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(info.getScope()), globalId); + TopicQueueMappingInfo oldInfo = mqEndPoints.get(mq); + if (oldInfo == null || oldInfo.getEpoch() <= info.getEpoch()) { + mqEndPoints.put(mq, info); + } + } + } + + + //accomplish the static logic queues + for (int i = 0; i < maxTotalNums; i++) { + MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(scope), i); + if (!mqEndPoints.containsKey(mq)) { + mqEndPointsOfBroker.put(mq, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST); + } else { + mqEndPointsOfBroker.put(mq, mqEndPoints.get(mq).getBname()); + } + } + } + return mqEndPointsOfBroker; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RequestBuilder.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RequestBuilder.java new file mode 100644 index 0000000000..9fec087be9 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RequestBuilder.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; + +import java.util.HashMap; +import java.util.Map; + +public class RequestBuilder { + + private static Map requestCodeMap = new HashMap(); + static { + requestCodeMap.put(RequestCode.PULL_MESSAGE, PullMessageRequestHeader.class); + } + + public static RpcRequestHeader buildCommonRpcHeader(int requestCode, String destBrokerName) { + return buildCommonRpcHeader(requestCode, null, destBrokerName); + } + + public static RpcRequestHeader buildCommonRpcHeader(int requestCode, Boolean oneway, String destBrokerName) { + Class requestHeaderClass = requestCodeMap.get(requestCode); + if (requestHeaderClass == null) { + throw new UnsupportedOperationException("unknown " + requestCode); + } + try { + RpcRequestHeader requestHeader = (RpcRequestHeader) requestHeaderClass.newInstance(); + requestHeader.setOway(oneway); + requestHeader.setBname(destBrokerName); + return requestHeader; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } + + public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq) { + return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), null); + } + + public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq, Boolean logic) { + return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic); + } + + public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, Boolean oneway, MessageQueue mq, Boolean logic) { + return buildTopicQueueRequestHeader(requestCode, oneway, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic); + } + + public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, Boolean oneway, String destBrokerName, String topic, int queueId, Boolean logic) { + Class requestHeaderClass = requestCodeMap.get(requestCode); + if (requestHeaderClass == null) { + throw new UnsupportedOperationException("unknown " + requestCode); + } + try { + TopicQueueRequestHeader requestHeader = (TopicQueueRequestHeader) requestHeaderClass.newInstance(); + requestHeader.setOway(oneway); + requestHeader.setBname(destBrokerName); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + requestHeader.setLo(logic); + return requestHeader; + } catch (Throwable t) { + throw new RuntimeException(t); + } + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClient.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClient.java new file mode 100644 index 0000000000..7876fdfdfd --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClient.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.common.message.MessageQueue; + +import java.util.concurrent.Future; + +public interface RpcClient { + + + //common invoke paradigm, the logic remote addr is defined in "bname" field of request + //For oneway request, the sign is labeled in request, and do not need an another method named "invokeOneway" + //For one + Future invoke(RpcRequest request, long timeoutMs) throws RpcException; + + //For rocketmq, most requests are corresponded to MessageQueue + //And for LogicQueue, the broker name is mocked, the physical addr could only be defined by MessageQueue + Future invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException; + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientHook.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientHook.java new file mode 100644 index 0000000000..e3430b5e61 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientHook.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +public abstract class RpcClientHook { + + //if the return is not null, return it + public abstract RpcResponse beforeRequest(RpcRequest rpcRequest) throws RpcException; + + //if the return is not null, return it + public abstract RpcResponse afterResponse(RpcResponse rpcResponse) throws RpcException; + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientImpl.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientImpl.java new file mode 100644 index 0000000000..62e6ec1bf5 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientImpl.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import io.netty.util.concurrent.ImmediateEventExecutor; +import io.netty.util.concurrent.Promise; +import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +public class RpcClientImpl implements RpcClient { + + private ClientMetadata clientMetadata; + + private RemotingClient remotingClient; + + private List clientHookList = new ArrayList(); + + public RpcClientImpl(ClientMetadata clientMetadata, RemotingClient remotingClient) { + this.clientMetadata = clientMetadata; + this.remotingClient = remotingClient; + } + + public void registerHook(RpcClientHook hook) { + clientHookList.add(hook); + } + + @Override + public Future invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException { + String bname = clientMetadata.getBrokerNameFromMessageQueue(mq); + request.getHeader().setBname(bname); + return invoke(request, timeoutMs); + } + + + public Promise createResponseFuture() { + return ImmediateEventExecutor.INSTANCE.newPromise(); + } + + @Override + public Future invoke(RpcRequest request, long timeoutMs) throws RpcException { + if (clientHookList.size() > 0) { + for (RpcClientHook rpcClientHook: clientHookList) { + RpcResponse response = rpcClientHook.beforeRequest(request); + if (response != null) { + //For 1.6, there is not easy-to-use future impl + return createResponseFuture().setSuccess(response); + } + } + } + String addr = getBrokerAddrByNameOrException(request.getHeader().bname); + Promise rpcResponsePromise = null; + try { + switch (request.getCode()) { + case RequestCode.PULL_MESSAGE: + rpcResponsePromise = handlePullMessage(addr, request, timeoutMs); + break; + case RequestCode.GET_MIN_OFFSET: + rpcResponsePromise = handleGetMinOffset(addr, request, timeoutMs); + break; + case RequestCode.GET_MAX_OFFSET: + rpcResponsePromise = handleGetMaxOffset(addr, request, timeoutMs); + break; + case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP: + rpcResponsePromise = handleSearchOffset(addr, request, timeoutMs); + break; + case RequestCode.GET_EARLIEST_MSG_STORETIME: + rpcResponsePromise = handleGetEarliestMsgStoretime(addr, request, timeoutMs); + break; + case RequestCode.QUERY_CONSUMER_OFFSET: + rpcResponsePromise = handleQueryConsumerOffset(addr, request, timeoutMs); + break; + case RequestCode.GET_TOPIC_STATS_INFO: + rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicStatsTable.class); + break; + case RequestCode.GET_TOPIC_CONFIG: + rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicConfigAndQueueMapping.class); + break; + default: + throw new RpcException(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, "Unknown request code " + request.getCode()); + } + } catch (RpcException rpcException) { + throw rpcException; + } catch (Exception e) { + throw new RpcException(ResponseCode.RPC_UNKNOWN, "error from remoting layer", e); + } + return rpcResponsePromise; + } + + + private String getBrokerAddrByNameOrException(String bname) throws RpcException { + String addr = this.clientMetadata.findMasterBrokerAddr(bname); + if (addr == null) { + throw new RpcException(ResponseCode.SYSTEM_ERROR, "cannot find addr for broker " + bname); + } + return addr; + } + + + private void processFailedResponse(String addr, RemotingCommand requestCommand, ResponseFuture responseFuture, Promise rpcResponsePromise) { + RemotingCommand responseCommand = responseFuture.getResponseCommand(); + if (responseCommand != null) { + //this should not happen + return; + } + int errorCode = ResponseCode.RPC_UNKNOWN; + String errorMessage = null; + if (!responseFuture.isSendRequestOK()) { + errorCode = ResponseCode.RPC_SEND_TO_CHANNEL_FAILED; + errorMessage = "send request failed to " + addr + ". Request: " + requestCommand; + } else if (responseFuture.isTimeout()) { + errorCode = ResponseCode.RPC_TIME_OUT; + errorMessage = "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + requestCommand; + } else { + errorMessage = "unknown reason. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; + } + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(errorCode, errorMessage))); + } + + + public Promise handlePullMessage(final String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + + final Promise rpcResponsePromise = createResponseFuture(); + + InvokeCallback callback = new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + RemotingCommand responseCommand = responseFuture.getResponseCommand(); + if (responseCommand == null) { + processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); + return; + } + try { + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: + case ResponseCode.PULL_NOT_FOUND: + case ResponseCode.PULL_RETRY_IMMEDIATELY: + case ResponseCode.PULL_OFFSET_MOVED: + PullMessageResponseHeader responseHeader = + (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + default: + RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); + rpcResponsePromise.setSuccess(rpcResponse); + + } + } catch (Exception e) { + String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; + RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); + rpcResponsePromise.setSuccess(rpcResponse); + } + } + }; + + this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); + return rpcResponsePromise; + } + + public Promise handleSearchOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + SearchOffsetResponseHeader responseHeader = + (SearchOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + + + + public Promise handleQueryConsumerOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + break; + } + case ResponseCode.QUERY_NOT_FOUND: { + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), null, null)); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + + public Promise handleCommonBodyRequest(final String addr, RpcRequest rpcRequest, long timeoutMillis, Class bodyClass) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + rpcResponsePromise.setSuccess(new RpcResponse(ResponseCode.SUCCESS, null, RemotingSerializable.decode(responseCommand.getBody(), bodyClass))); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + + public Promise handleGetMinOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + GetMinOffsetResponseHeader responseHeader = + (GetMinOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + + public Promise handleGetMaxOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + GetMaxOffsetResponseHeader responseHeader = + (GetMaxOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + + public Promise handleGetEarliestMsgStoretime(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception { + final Promise rpcResponsePromise = createResponseFuture(); + + RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest); + + RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis); + assert responseCommand != null; + switch (responseCommand.getCode()) { + case ResponseCode.SUCCESS: { + GetEarliestMsgStoretimeResponseHeader responseHeader = + (GetEarliestMsgStoretimeResponseHeader) responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class); + rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); + break; + } + default: { + rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error"))); + } + } + return rpcResponsePromise; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientUtils.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientUtils.java new file mode 100644 index 0000000000..40c6eef1cd --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientUtils.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.nio.ByteBuffer; + +public class RpcClientUtils { + + public static RemotingCommand createCommandForRpcRequest(RpcRequest rpcRequest) { + RemotingCommand cmd = RemotingCommand.createRequestCommand(rpcRequest.getCode(), rpcRequest.getHeader()); + cmd.setBody(encodeBody(rpcRequest.getBody())); + return cmd; + } + + public static RemotingCommand createCommandForRpcResponse(RpcResponse rpcResponse) { + RemotingCommand cmd = RemotingCommand.createResponseCommandWithHeader(rpcResponse.getCode(), rpcResponse.getHeader()); + cmd.setRemark(rpcResponse.getException() == null ? "" : rpcResponse.getException().getMessage()); + cmd.setBody(encodeBody(rpcResponse.getBody())); + return cmd; + } + + public static byte[] encodeBody(Object body) { + if (body == null) { + return null; + } + if (body instanceof byte[]) { + return (byte[])body; + } else if (body instanceof RemotingSerializable) { + return ((RemotingSerializable) body).encode(); + } else if (body instanceof ByteBuffer) { + ByteBuffer buffer = (ByteBuffer)body; + buffer.mark(); + byte[] data = new byte[buffer.remaining()]; + buffer.get(data); + buffer.reset(); + return data; + } else { + throw new RuntimeException("Unsupported body type " + body.getClass()); + } + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcException.java similarity index 53% rename from common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java rename to common/src/main/java/org/apache/rocketmq/common/rpc/RpcException.java index edb521f3d0..36fc0568a3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcException.java @@ -14,36 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.common.protocol.body; +package org.apache.rocketmq.common.rpc; -import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.apache.rocketmq.remoting.exception.RemotingException; -public class SealTopicLogicalQueueRequestBody extends RemotingSerializable { - private String topic; - private int queueId; - private int logicalQueueIndex; - - public String getTopic() { - return topic; +public class RpcException extends RemotingException { + private int errorCode; + public RpcException(int errorCode, String message) { + super(message); + this.errorCode = errorCode; } - public void setTopic(String topic) { - this.topic = topic; + public RpcException(int errorCode, String message, Throwable cause) { + super(message, cause); + this.errorCode = errorCode; } - public int getQueueId() { - return queueId; + public int getErrorCode() { + return errorCode; } - public void setQueueId(int queueId) { - this.queueId = queueId; - } - - public int getLogicalQueueIndex() { - return logicalQueueIndex; - } - - public void setLogicalQueueIndex(int logicalQueueIndex) { - this.logicalQueueIndex = logicalQueueIndex; + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequest.java similarity index 59% rename from common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java rename to common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequest.java index b7e0c46d05..90bb696e37 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequest.java @@ -14,24 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.rocketmq.common.protocol.header; +package org.apache.rocketmq.common.rpc; -import org.apache.rocketmq.remoting.CommandCustomHeader; -import org.apache.rocketmq.remoting.annotation.CFNotNull; -import org.apache.rocketmq.remoting.exception.RemotingCommandException; +public class RpcRequest { + int code; + private RpcRequestHeader header; + private Object body; -public class QueryTopicLogicalQueueMappingRequestHeader implements CommandCustomHeader { - @CFNotNull - private String topic; - - @Override public void checkFields() throws RemotingCommandException { + public RpcRequest(int code, RpcRequestHeader header, Object body) { + this.code = code; + this.header = header; + this.body = body; } - public String getTopic() { - return topic; + public RpcRequestHeader getHeader() { + return header; } - public void setTopic(String topic) { - this.topic = topic; + public Object getBody() { + return body; + } + + public int getCode() { + return code; } } diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequestHeader.java new file mode 100644 index 0000000000..593df81549 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequestHeader.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.remoting.CommandCustomHeader; + +public abstract class RpcRequestHeader implements CommandCustomHeader { + //the namespace name + protected String ns; + //if the data has been namespaced + protected Boolean nsd; + //the abstract remote addr name, usually the physical broker name + protected String bname; + //oneway + protected Boolean oway; + + public String getBname() { + return bname; + } + + public void setBname(String bname) { + this.bname = bname; + } + + public Boolean getOway() { + return oway; + } + + public void setOway(Boolean oway) { + this.oway = oway; + } + + public String getNs() { + return ns; + } + + public void setNs(String ns) { + this.ns = ns; + } + + public Boolean getNsd() { + return nsd; + } + + public void setNsd(Boolean nsd) { + this.nsd = nsd; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/RpcResponse.java b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcResponse.java new file mode 100644 index 0000000000..5155bd241c --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/RpcResponse.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +import org.apache.rocketmq.remoting.CommandCustomHeader; + +public class RpcResponse { + private int code; + private CommandCustomHeader header; + private Object body; + public RpcException exception; + + public RpcResponse() { + + } + + public RpcResponse(int code, CommandCustomHeader header, Object body) { + this.code = code; + this.header = header; + this.body = body; + } + + public RpcResponse(RpcException rpcException) { + this.code = rpcException.getErrorCode(); + this.exception = rpcException; + } + + public int getCode() { + return code; + } + + public CommandCustomHeader getHeader() { + return header; + } + + public void setHeader(CommandCustomHeader header) { + this.header = header; + } + + public Object getBody() { + return body; + } + + public void setBody(Object body) { + this.body = body; + } + + public RpcException getException() { + return exception; + } + + public void setException(RpcException exception) { + this.exception = exception; + } + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/TopicQueueRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/rpc/TopicQueueRequestHeader.java new file mode 100644 index 0000000000..660f046e14 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/TopicQueueRequestHeader.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +public abstract class TopicQueueRequestHeader extends TopicRequestHeader { + + public abstract Integer getQueueId(); + public abstract void setQueueId(Integer queueId); + +} diff --git a/common/src/main/java/org/apache/rocketmq/common/rpc/TopicRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/rpc/TopicRequestHeader.java new file mode 100644 index 0000000000..a70cded67b --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/rpc/TopicRequestHeader.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.rpc; + +public abstract class TopicRequestHeader extends RpcRequestHeader { + //logical + protected Boolean lo; + + public abstract String getTopic(); + public abstract void setTopic(String topic); + + public Boolean getLo() { + return lo; + } + public void setLo(Boolean lo) { + this.lo = lo; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/LogicQueueMappingItem.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/LogicQueueMappingItem.java new file mode 100644 index 0000000000..3c217f5727 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/LogicQueueMappingItem.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +public class LogicQueueMappingItem extends RemotingSerializable { + + private int gen; // immutable + private int queueId; //, immutable + private String bname; //important, immutable + private long logicOffset; // the start of the logic offset, important, can be changed by command only once + private long startOffset; // the start of the physical offset, should always be 0, immutable + private long endOffset = -1; // the end of the physical offset, excluded, revered -1, mutable + private long timeOfStart = -1; // mutable, reserved + private long timeOfEnd = -1; // mutable, reserved + + //make sure it has a default constructor + public LogicQueueMappingItem() { + + } + + public LogicQueueMappingItem(int gen, int queueId, String bname, long logicOffset, long startOffset, long endOffset, long timeOfStart, long timeOfEnd) { + this.gen = gen; + this.queueId = queueId; + this.bname = bname; + this.logicOffset = logicOffset; + this.startOffset = startOffset; + this.endOffset = endOffset; + this.timeOfStart = timeOfStart; + this.timeOfEnd = timeOfEnd; + } + + + //should only be user in sendMessage and getMinOffset + public long computeStaticQueueOffsetLoosely(long physicalQueueOffset) { + //consider the newly mapped item + if (logicOffset < 0) { + return -1; + } + if (physicalQueueOffset < startOffset) { + return logicOffset; + } + if (endOffset >= startOffset + && endOffset < physicalQueueOffset) { + return logicOffset + (endOffset - startOffset); + } + return logicOffset + (physicalQueueOffset - startOffset); + } + + public long computeStaticQueueOffsetStrictly(long physicalQueueOffset) { + assert logicOffset >= 0; + + if (physicalQueueOffset < startOffset) { + return logicOffset; + } + return logicOffset + (physicalQueueOffset - startOffset); + } + + public long computePhysicalQueueOffset(long staticQueueOffset) { + return (staticQueueOffset - logicOffset) + startOffset; + } + + public long computeMaxStaticQueueOffset() { + if (endOffset >= startOffset) { + return logicOffset + endOffset - startOffset; + } else { + return logicOffset; + } + } + public boolean checkIfEndOffsetDecided() { + //if the endOffset == startOffset, then the item should be deleted + return endOffset > startOffset; + } + + public boolean checkIfLogicoffsetDecided() { + return logicOffset >= 0; + } + + public long computeOffsetDelta() { + return logicOffset - startOffset; + } + + public int getGen() { + return gen; + } + + public int getQueueId() { + return queueId; + } + + public String getBname() { + return bname; + } + + public long getLogicOffset() { + return logicOffset; + } + + public long getStartOffset() { + return startOffset; + } + + public long getEndOffset() { + return endOffset; + } + + public long getTimeOfStart() { + return timeOfStart; + } + + public long getTimeOfEnd() { + return timeOfEnd; + } + + public void setLogicOffset(long logicOffset) { + this.logicOffset = logicOffset; + } + + public void setEndOffset(long endOffset) { + this.endOffset = endOffset; + } + + public void setTimeOfStart(long timeOfStart) { + this.timeOfStart = timeOfStart; + } + + public void setTimeOfEnd(long timeOfEnd) { + this.timeOfEnd = timeOfEnd; + } + + public void setGen(int gen) { + this.gen = gen; + } + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + public void setBname(String bname) { + this.bname = bname; + } + + public void setStartOffset(long startOffset) { + this.startOffset = startOffset; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (!(o instanceof LogicQueueMappingItem)) return false; + + LogicQueueMappingItem item = (LogicQueueMappingItem) o; + + return new EqualsBuilder() + .append(gen, item.gen) + .append(queueId, item.queueId) + .append(logicOffset, item.logicOffset) + .append(startOffset, item.startOffset) + .append(endOffset, item.endOffset) + .append(timeOfStart, item.timeOfStart) + .append(timeOfEnd, item.timeOfEnd) + .append(bname, item.bname) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(gen) + .append(queueId) + .append(bname) + .append(logicOffset) + .append(startOffset) + .append(endOffset) + .append(timeOfStart) + .append(timeOfEnd) + .toHashCode(); + } + + @Override + public String toString() { + return "LogicQueueMappingItem{" + + "gen=" + gen + + ", queueId=" + queueId + + ", bname='" + bname + '\'' + + ", logicOffset=" + logicOffset + + ", startOffset=" + startOffset + + ", endOffset=" + endOffset + + ", timeOfStart=" + timeOfStart + + ", timeOfEnd=" + timeOfEnd + + '}'; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicConfigAndQueueMapping.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicConfigAndQueueMapping.java new file mode 100644 index 0000000000..cef6418bda --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicConfigAndQueueMapping.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.rocketmq.common.TopicConfig; + +public class TopicConfigAndQueueMapping extends TopicConfig { + private TopicQueueMappingDetail mappingDetail; + + public TopicConfigAndQueueMapping() { + } + + public TopicConfigAndQueueMapping(TopicConfig topicConfig, TopicQueueMappingDetail mappingDetail) { + super(topicConfig); + this.mappingDetail = mappingDetail; + } + + public TopicQueueMappingDetail getMappingDetail() { + return mappingDetail; + } + + public void setMappingDetail(TopicQueueMappingDetail mappingDetail) { + this.mappingDetail = mappingDetail; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (!(o instanceof TopicConfigAndQueueMapping)) return false; + + TopicConfigAndQueueMapping that = (TopicConfigAndQueueMapping) o; + + return new EqualsBuilder() + .appendSuper(super.equals(o)) + .append(mappingDetail, that.mappingDetail) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .appendSuper(super.hashCode()) + .append(mappingDetail) + .toHashCode(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingContext.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingContext.java new file mode 100644 index 0000000000..d6a6fd9764 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingContext.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +public class TopicQueueMappingContext { + private String topic; + private Integer globalId; + private TopicQueueMappingDetail mappingDetail; + private List mappingItemList; + private LogicQueueMappingItem leaderItem; + + private LogicQueueMappingItem currentItem; + + public TopicQueueMappingContext(String topic, Integer globalId, TopicQueueMappingDetail mappingDetail, List mappingItemList, LogicQueueMappingItem leaderItem) { + this.topic = topic; + this.globalId = globalId; + this.mappingDetail = mappingDetail; + this.mappingItemList = mappingItemList; + this.leaderItem = leaderItem; + + } + + + public boolean isLeader() { + return leaderItem != null && leaderItem.getBname().equals(mappingDetail.getBname()); + } + + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public Integer getGlobalId() { + return globalId; + } + + public void setGlobalId(Integer globalId) { + this.globalId = globalId; + } + + + public TopicQueueMappingDetail getMappingDetail() { + return mappingDetail; + } + + public void setMappingDetail(TopicQueueMappingDetail mappingDetail) { + this.mappingDetail = mappingDetail; + } + + public List getMappingItemList() { + return mappingItemList; + } + + public void setMappingItemList(ImmutableList mappingItemList) { + this.mappingItemList = mappingItemList; + } + + public LogicQueueMappingItem getLeaderItem() { + return leaderItem; + } + + public void setLeaderItem(LogicQueueMappingItem leaderItem) { + this.leaderItem = leaderItem; + } + + public LogicQueueMappingItem getCurrentItem() { + return currentItem; + } + + public void setCurrentItem(LogicQueueMappingItem currentItem) { + this.currentItem = currentItem; + } + + public void setMappingItemList(List mappingItemList) { + this.mappingItemList = mappingItemList; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingDetail.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingDetail.java new file mode 100644 index 0000000000..f0f27a5cd0 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingDetail.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class TopicQueueMappingDetail extends TopicQueueMappingInfo { + + // the mapping info in current broker, do not register to nameserver + // make sure this value is not null + private ConcurrentMap> hostedQueues = new ConcurrentHashMap>(); + + //make sure there is a default constructor + public TopicQueueMappingDetail() { + + } + + public TopicQueueMappingDetail(String topic, int totalQueues, String bname, long epoch) { + super(topic, totalQueues, bname, epoch); + } + + + + public static boolean putMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId, List mappingInfo) { + if (mappingInfo.isEmpty()) { + return true; + } + mappingDetail.hostedQueues.put(globalId, mappingInfo); + return true; + } + + public static List getMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId) { + return mappingDetail.hostedQueues.get(globalId); + } + + public static ConcurrentMap buildIdMap(TopicQueueMappingDetail mappingDetail, int level) { + //level 0 means current leader in this broker + //level 1 means previous leader in this broker, reserved for + assert level == LEVEL_0 ; + + if (mappingDetail.hostedQueues == null || mappingDetail.hostedQueues.isEmpty()) { + return new ConcurrentHashMap(); + } + ConcurrentMap tmpIdMap = new ConcurrentHashMap(); + for (Map.Entry> entry: mappingDetail.hostedQueues.entrySet()) { + Integer globalId = entry.getKey(); + List items = entry.getValue(); + if (level == LEVEL_0 + && items.size() >= 1) { + LogicQueueMappingItem curr = items.get(items.size() - 1); + if (mappingDetail.bname.equals(curr.getBname())) { + tmpIdMap.put(globalId, curr.getQueueId()); + } + } + } + return tmpIdMap; + } + + + public static long computeMaxOffsetFromMapping(TopicQueueMappingDetail mappingDetail, Integer globalId) { + List mappingItems = getMappingInfo(mappingDetail, globalId); + if (mappingItems == null + || mappingItems.isEmpty()) { + return -1; + } + LogicQueueMappingItem item = mappingItems.get(mappingItems.size() - 1); + return item.computeMaxStaticQueueOffset(); + } + + + public static TopicQueueMappingInfo cloneAsMappingInfo(TopicQueueMappingDetail mappingDetail) { + TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(mappingDetail.topic, mappingDetail.totalQueues, mappingDetail.bname, mappingDetail.epoch); + topicQueueMappingInfo.currIdMap = TopicQueueMappingDetail.buildIdMap(mappingDetail, LEVEL_0); + return topicQueueMappingInfo; + } + + public static boolean checkIfAsPhysical(TopicQueueMappingDetail mappingDetail, Integer globalId) { + List mappingItems = getMappingInfo(mappingDetail, globalId); + return mappingItems == null + || (mappingItems.size() == 1 + && mappingItems.get(0).getLogicOffset() == 0); + } + + public ConcurrentMap> getHostedQueues() { + return hostedQueues; + } + + public void setHostedQueues(ConcurrentMap> hostedQueues) { + this.hostedQueues = hostedQueues; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (!(o instanceof TopicQueueMappingDetail)) return false; + + TopicQueueMappingDetail that = (TopicQueueMappingDetail) o; + + return new EqualsBuilder() + .append(hostedQueues, that.hostedQueues) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(hostedQueues) + .toHashCode(); + } + + @Override + public String toString() { + return "TopicQueueMappingDetail{" + + "hostedQueues=" + hostedQueues + + ", topic='" + topic + '\'' + + ", totalQueues=" + totalQueues + + ", bname='" + bname + '\'' + + ", epoch=" + epoch + + ", dirty=" + dirty + + ", currIdMap=" + currIdMap + + '}'; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingInfo.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingInfo.java new file mode 100644 index 0000000000..784247d2da --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingInfo.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class TopicQueueMappingInfo extends RemotingSerializable { + public static final int LEVEL_0 = 0; + + String topic; // redundant field + String scope = MixAll.METADATA_SCOPE_GLOBAL; + int totalQueues; + String bname; //identify the hosted broker name + long epoch; //important to fence the old dirty data + boolean dirty; //indicate if the data is dirty + //register to broker to construct the route + protected ConcurrentMap currIdMap = new ConcurrentHashMap(); + + public TopicQueueMappingInfo() { + + } + + public TopicQueueMappingInfo(String topic, int totalQueues, String bname, long epoch) { + this.topic = topic; + this.totalQueues = totalQueues; + this.bname = bname; + this.epoch = epoch; + this.dirty = false; + } + + public boolean isDirty() { + return dirty; + } + + public void setDirty(boolean dirty) { + this.dirty = dirty; + } + + public int getTotalQueues() { + return totalQueues; + } + + + public String getBname() { + return bname; + } + + public String getTopic() { + return topic; + } + + public long getEpoch() { + return epoch; + } + + public void setEpoch(long epoch) { + this.epoch = epoch; + } + + public void setTotalQueues(int totalQueues) { + this.totalQueues = totalQueues; + } + + public ConcurrentMap getCurrIdMap() { + return currIdMap; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public void setBname(String bname) { + this.bname = bname; + } + + public void setCurrIdMap(ConcurrentMap currIdMap) { + this.currIdMap = currIdMap; + } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TopicQueueMappingInfo)) return false; + + TopicQueueMappingInfo info = (TopicQueueMappingInfo) o; + + if (totalQueues != info.totalQueues) return false; + if (epoch != info.epoch) return false; + if (dirty != info.dirty) return false; + if (topic != null ? !topic.equals(info.topic) : info.topic != null) return false; + if (scope != null ? !scope.equals(info.scope) : info.scope != null) return false; + if (bname != null ? !bname.equals(info.bname) : info.bname != null) return false; + return currIdMap != null ? currIdMap.equals(info.currIdMap) : info.currIdMap == null; + } + + @Override + public int hashCode() { + int result = topic != null ? topic.hashCode() : 0; + result = 31 * result + (scope != null ? scope.hashCode() : 0); + result = 31 * result + totalQueues; + result = 31 * result + (bname != null ? bname.hashCode() : 0); + result = 31 * result + (int) (epoch ^ (epoch >>> 32)); + result = 31 * result + (dirty ? 1 : 0); + result = 31 * result + (currIdMap != null ? currIdMap.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "TopicQueueMappingInfo{" + + "topic='" + topic + '\'' + + ", scope='" + scope + '\'' + + ", totalQueues=" + totalQueues + + ", bname='" + bname + '\'' + + ", epoch=" + epoch + + ", dirty=" + dirty + + ", currIdMap=" + currIdMap + + '}'; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingOne.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingOne.java new file mode 100644 index 0000000000..636f1d5098 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingOne.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.List; + +public class TopicQueueMappingOne extends RemotingSerializable { + + String topic; // redundant field + String bname; //identify the hosted broker name + Integer globalId; + List items; + TopicQueueMappingDetail mappingDetail; + + public TopicQueueMappingOne(TopicQueueMappingDetail mappingDetail, String topic, String bname, Integer globalId, List items) { + this.mappingDetail = mappingDetail; + this.topic = topic; + this.bname = bname; + this.globalId = globalId; + this.items = items; + } + + public String getTopic() { + return topic; + } + + public String getBname() { + return bname; + } + + public Integer getGlobalId() { + return globalId; + } + + public List getItems() { + return items; + } + + public TopicQueueMappingDetail getMappingDetail() { + return mappingDetail; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (!(o instanceof TopicQueueMappingOne)) + return false; + + TopicQueueMappingOne that = (TopicQueueMappingOne) o; + + if (topic != null ? !topic.equals(that.topic) : that.topic != null) + return false; + if (bname != null ? !bname.equals(that.bname) : that.bname != null) + return false; + if (globalId != null ? !globalId.equals(that.globalId) : that.globalId != null) + return false; + if (items != null ? !items.equals(that.items) : that.items != null) + return false; + return mappingDetail != null ? mappingDetail.equals(that.mappingDetail) : that.mappingDetail == null; + } + + @Override public int hashCode() { + int result = topic != null ? topic.hashCode() : 0; + result = 31 * result + (bname != null ? bname.hashCode() : 0); + result = 31 * result + (globalId != null ? globalId.hashCode() : 0); + result = 31 * result + (items != null ? items.hashCode() : 0); + result = 31 * result + (mappingDetail != null ? mappingDetail.hashCode() : 0); + return result; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtils.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtils.java new file mode 100644 index 0000000000..bf02ccd457 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtils.java @@ -0,0 +1,694 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; + +import java.io.File; +import java.util.AbstractMap; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Random; +import java.util.Set; + +public class TopicQueueMappingUtils { + + public static final int DEFAULT_BLOCK_SEQ_SIZE = 10000; + + public static class MappingAllocator { + Map brokerNumMap = new HashMap(); + Map idToBroker = new HashMap(); + //used for remapping + Map brokerNumMapBeforeRemapping = null; + int currentIndex = 0; + Random random = new Random(); + List leastBrokers = new ArrayList(); + private MappingAllocator(Map idToBroker, Map brokerNumMap, Map brokerNumMapBeforeRemapping) { + this.idToBroker.putAll(idToBroker); + this.brokerNumMap.putAll(brokerNumMap); + this.brokerNumMapBeforeRemapping = brokerNumMapBeforeRemapping; + } + + private void freshState() { + int minNum = Integer.MAX_VALUE; + for (Map.Entry entry : brokerNumMap.entrySet()) { + if (entry.getValue() < minNum) { + leastBrokers.clear(); + leastBrokers.add(entry.getKey()); + minNum = entry.getValue(); + } else if (entry.getValue() == minNum) { + leastBrokers.add(entry.getKey()); + } + } + //reduce the remapping + if (brokerNumMapBeforeRemapping != null + && !brokerNumMapBeforeRemapping.isEmpty()) { + Collections.sort(leastBrokers, new Comparator() { + @Override + public int compare(String o1, String o2) { + int i1 = 0, i2 = 0; + if (brokerNumMapBeforeRemapping.containsKey(o1)) { + i1 = brokerNumMapBeforeRemapping.get(o1); + } + if (brokerNumMapBeforeRemapping.containsKey(o2)) { + i2 = brokerNumMapBeforeRemapping.get(o2); + } + return i1 - i2; + } + }); + } else { + //reduce the imbalance + Collections.shuffle(leastBrokers); + } + currentIndex = leastBrokers.size() - 1; + } + private String nextBroker() { + if (leastBrokers.isEmpty()) { + freshState(); + } + int tmpIndex = currentIndex % leastBrokers.size(); + return leastBrokers.remove(tmpIndex); + } + + public Map getBrokerNumMap() { + return brokerNumMap; + } + + public void upToNum(int maxQueueNum) { + int currSize = idToBroker.size(); + if (maxQueueNum <= currSize) { + return; + } + for (int i = currSize; i < maxQueueNum; i++) { + String nextBroker = nextBroker(); + if (brokerNumMap.containsKey(nextBroker)) { + brokerNumMap.put(nextBroker, brokerNumMap.get(nextBroker) + 1); + } else { + brokerNumMap.put(nextBroker, 1); + } + idToBroker.put(i, nextBroker); + } + } + + public Map getIdToBroker() { + return idToBroker; + } + } + + + public static MappingAllocator buildMappingAllocator(Map idToBroker, Map brokerNumMap, Map brokerNumMapBeforeRemapping) { + return new MappingAllocator(idToBroker, brokerNumMap, brokerNumMapBeforeRemapping); + } + + public static Map.Entry findMaxEpochAndQueueNum(List mappingDetailList) { + long epoch = -1; + int queueNum = 0; + for (TopicQueueMappingDetail mappingDetail : mappingDetailList) { + if (mappingDetail.getEpoch() > epoch) { + epoch = mappingDetail.getEpoch(); + } + if (mappingDetail.getTotalQueues() > queueNum) { + queueNum = mappingDetail.getTotalQueues(); + } + } + return new AbstractMap.SimpleImmutableEntry(epoch, queueNum); + } + + public static List getMappingDetailFromConfig(Collection configs) { + List detailList = new ArrayList(); + for (TopicConfigAndQueueMapping configMapping : configs) { + if (configMapping.getMappingDetail() != null) { + detailList.add(configMapping.getMappingDetail()); + } + } + return detailList; + } + + public static Map.Entry checkNameEpochNumConsistence(String topic, Map brokerConfigMap) { + if (brokerConfigMap == null + || brokerConfigMap.isEmpty()) { + return null; + } + //make sure it it not null + long maxEpoch = -1; + int maxNum = -1; + String scope = null; + for (Map.Entry entry : brokerConfigMap.entrySet()) { + String broker = entry.getKey(); + TopicConfigAndQueueMapping configMapping = entry.getValue(); + if (configMapping.getMappingDetail() == null) { + throw new RuntimeException("Mapping info should not be null in broker " + broker); + } + TopicQueueMappingDetail mappingDetail = configMapping.getMappingDetail(); + if (!broker.equals(mappingDetail.getBname())) { + throw new RuntimeException(String.format("The broker name is not equal %s != %s ", broker, mappingDetail.getBname())); + } + if (mappingDetail.isDirty()) { + throw new RuntimeException("The mapping info is dirty in broker " + broker); + } + if (!configMapping.getTopicName().equals(mappingDetail.getTopic())) { + throw new RuntimeException("The topic name is inconsistent in broker " + broker); + } + if (topic != null + && !topic.equals(mappingDetail.getTopic())) { + throw new RuntimeException("The topic name is not match for broker " + broker); + } + + if (scope != null + && !scope.equals(mappingDetail.getScope())) { + throw new RuntimeException(String.format("scope dose not match %s != %s in %s", mappingDetail.getScope(), scope, broker)); + } else { + scope = mappingDetail.getScope(); + } + + if (maxEpoch != -1 + && maxEpoch != mappingDetail.getEpoch()) { + throw new RuntimeException(String.format("epoch dose not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname())); + } else { + maxEpoch = mappingDetail.getEpoch(); + } + + if (maxNum != -1 + && maxNum != mappingDetail.getTotalQueues()) { + throw new RuntimeException(String.format("total queue number dose not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname())); + } else { + maxNum = mappingDetail.getTotalQueues(); + } + } + return new AbstractMap.SimpleEntry(maxEpoch, maxNum); + } + + public static String getMockBrokerName(String scope) { + assert scope != null; + if (scope.equals(MixAll.METADATA_SCOPE_GLOBAL)) { + return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope.substring(2); + } else { + return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope; + } + } + + public static void makeSureLogicQueueMappingItemImmutable(List oldItems, List newItems, boolean epochEqual, boolean isCLean) { + if (oldItems == null || oldItems.isEmpty()) { + return; + } + if (newItems == null || newItems.isEmpty()) { + throw new RuntimeException("The new item list is null or empty"); + } + int iold = 0, inew = 0; + while (iold < oldItems.size() && inew < newItems.size()) { + LogicQueueMappingItem newItem = newItems.get(inew); + LogicQueueMappingItem oldItem = oldItems.get(iold); + if (newItem.getGen() < oldItem.getGen()) { + //the earliest item may have been deleted concurrently + inew++; + } else if (oldItem.getGen() < newItem.getGen()) { + //in the following cases, the new item-list has less items than old item-list + //1. the queue is mapped back to a broker which hold the logic queue before + //2. The earliest item is deleted by TopicQueueMappingCleanService + iold++; + } else { + assert oldItem.getBname().equals(newItem.getBname()); + assert oldItem.getQueueId() == newItem.getQueueId(); + assert oldItem.getStartOffset() == newItem.getStartOffset(); + if (oldItem.getLogicOffset() != -1) { + assert oldItem.getLogicOffset() == newItem.getLogicOffset(); + } + iold++; + inew++; + } + } + if (epochEqual) { + LogicQueueMappingItem oldLeader = oldItems.get(oldItems.size() - 1); + LogicQueueMappingItem newLeader = newItems.get(newItems.size() - 1); + if (newLeader.getGen() != oldLeader.getGen() + || !newLeader.getBname().equals(oldLeader.getBname()) + || newLeader.getQueueId() != oldLeader.getQueueId() + || newLeader.getStartOffset() != oldLeader.getStartOffset()) { + throw new RuntimeException("The new leader is different but epoch equal"); + } + } + } + + + public static void checkLogicQueueMappingItemOffset(List items) { + if (items == null + || items.isEmpty()) { + return; + } + int lastGen = -1; + long lastOffset = -1; + for (int i = items.size() - 1; i >= 0 ; i--) { + LogicQueueMappingItem item = items.get(i); + if (item.getStartOffset() < 0 + || item.getGen() < 0 + || item.getQueueId() < 0) { + throw new RuntimeException("The field is illegal, should not be negative"); + } + if (items.size() >= 2 + && i <= items.size() - 2 + && items.get(i).getLogicOffset() < 0) { + throw new RuntimeException("The non-latest item has negative logic offset"); + } + if (lastGen != -1 && item.getGen() >= lastGen) { + throw new RuntimeException("The gen dose not increase monotonically"); + } + + if (item.getEndOffset() != -1 + && item.getEndOffset() < item.getStartOffset()) { + throw new RuntimeException("The endOffset is smaller than the start offset"); + } + + if (lastOffset != -1 && item.getLogicOffset() != -1) { + if (item.getLogicOffset() >= lastOffset) { + throw new RuntimeException("The base logic offset dose not increase monotonically"); + } + if (item.computeMaxStaticQueueOffset() >= lastOffset) { + throw new RuntimeException("The max logic offset dose not increase monotonically"); + } + } + lastGen = item.getGen(); + lastOffset = item.getLogicOffset(); + } + } + + public static void checkIfReusePhysicalQueue(Collection mappingOnes) { + Map physicalQueueIdMap = new HashMap(); + for (TopicQueueMappingOne mappingOne : mappingOnes) { + for (LogicQueueMappingItem item: mappingOne.items) { + String physicalQueueId = item.getBname() + "-" + item.getQueueId(); + if (physicalQueueIdMap.containsKey(physicalQueueId)) { + throw new RuntimeException(String.format("Topic %s global queue id %d and %d shared the same physical queue %s", + mappingOne.topic, mappingOne.globalId, physicalQueueIdMap.get(physicalQueueId).globalId, physicalQueueId)); + } else { + physicalQueueIdMap.put(physicalQueueId, mappingOne); + } + } + } + } + + public static void checkLeaderInTargetBrokers(Collection mappingOnes, Set targetBrokers) { + for (TopicQueueMappingOne mappingOne : mappingOnes) { + if (!targetBrokers.contains(mappingOne.bname)) { + throw new RuntimeException("The leader broker does not in target broker"); + } + } + } + + public static void checkPhysicalQueueConsistence(Map brokerConfigMap) { + for (Map.Entry entry : brokerConfigMap.entrySet()) { + TopicConfigAndQueueMapping configMapping = entry.getValue(); + assert configMapping != null; + assert configMapping.getMappingDetail() != null; + if (configMapping.getReadQueueNums() < configMapping.getWriteQueueNums()) { + throw new RuntimeException("Read queues is smaller than write queues"); + } + for (List items: configMapping.getMappingDetail().getHostedQueues().values()) { + for (LogicQueueMappingItem item: items) { + if (item.getStartOffset() != 0) { + throw new RuntimeException("The start offset dose not begin from 0"); + } + TopicConfig topicConfig = brokerConfigMap.get(item.getBname()); + if (topicConfig == null) { + throw new RuntimeException("The broker of item dose not exist"); + } + if (item.getQueueId() >= topicConfig.getWriteQueueNums()) { + throw new RuntimeException("The physical queue id is overflow the write queues"); + } + } + } + } + } + + + + public static Map checkAndBuildMappingItems(List mappingDetailList, boolean replace, boolean checkConsistence) { + Collections.sort(mappingDetailList, new Comparator() { + @Override + public int compare(TopicQueueMappingDetail o1, TopicQueueMappingDetail o2) { + return (int)(o2.getEpoch() - o1.getEpoch()); + } + }); + + int maxNum = 0; + Map globalIdMap = new HashMap(); + for (TopicQueueMappingDetail mappingDetail : mappingDetailList) { + if (mappingDetail.totalQueues > maxNum) { + maxNum = mappingDetail.totalQueues; + } + for (Map.Entry> entry : mappingDetail.getHostedQueues().entrySet()) { + Integer globalid = entry.getKey(); + checkLogicQueueMappingItemOffset(entry.getValue()); + String leaderBrokerName = getLeaderBroker(entry.getValue()); + if (!leaderBrokerName.equals(mappingDetail.getBname())) { + //not the leader + continue; + } + if (globalIdMap.containsKey(globalid)) { + if (!replace) { + throw new RuntimeException(String.format("The queue id is duplicated in broker %s %s", leaderBrokerName, mappingDetail.getBname())); + } + } else { + globalIdMap.put(globalid, new TopicQueueMappingOne(mappingDetail, mappingDetail.topic, mappingDetail.bname, globalid, entry.getValue())); + } + } + } + if (checkConsistence) { + if (maxNum != globalIdMap.size()) { + throw new RuntimeException(String.format("The total queue number in config dose not match the real hosted queues %d != %d", maxNum, globalIdMap.size())); + } + for (int i = 0; i < maxNum; i++) { + if (!globalIdMap.containsKey(i)) { + throw new RuntimeException(String.format("The queue number %s is not in globalIdMap", i)); + } + } + } + checkIfReusePhysicalQueue(globalIdMap.values()); + return globalIdMap; + } + + public static String getLeaderBroker(List items) { + return getLeaderItem(items).getBname(); + } + public static LogicQueueMappingItem getLeaderItem(List items) { + assert items.size() > 0; + return items.get(items.size() - 1); + } + + public static String writeToTemp(TopicRemappingDetailWrapper wrapper, boolean after) { + String topic = wrapper.getTopic(); + String data = wrapper.toJson(); + String suffix = TopicRemappingDetailWrapper.SUFFIX_BEFORE; + if (after) { + suffix = TopicRemappingDetailWrapper.SUFFIX_AFTER; + } + String fileName = System.getProperty("java.io.tmpdir") + File.separator + topic + "-" + wrapper.getEpoch() + suffix; + try { + MixAll.string2File(data, fileName); + return fileName; + } catch (Exception e) { + throw new RuntimeException("write file failed " + fileName,e); + } + } + + public static long blockSeqRoundUp(long offset, long blockSeqSize) { + long num = offset / blockSeqSize; + long left = offset % blockSeqSize; + if (left < blockSeqSize / 2) { + return (num + 1) * blockSeqSize; + } else { + return (num + 2) * blockSeqSize; + } + } + + public static void checkTargetBrokersComplete(Set targetBrokers, Map brokerConfigMap) { + for (String broker : brokerConfigMap.keySet()) { + if (brokerConfigMap.get(broker).getMappingDetail().getHostedQueues().isEmpty()) { + continue; + } + if (!targetBrokers.contains(broker)) { + throw new RuntimeException("The existed broker " + broker + " dose not in target brokers "); + } + } + } + + public static void checkNonTargetBrokers(Set targetBrokers, Set nonTargetBrokers) { + for (String broker : nonTargetBrokers) { + if (targetBrokers.contains(broker)) { + throw new RuntimeException("The non-target broker exist in target broker"); + } + } + } + + public static TopicRemappingDetailWrapper createTopicConfigMapping(String topic, int queueNum, Set targetBrokers, Map brokerConfigMap) { + checkTargetBrokersComplete(targetBrokers, brokerConfigMap); + Map globalIdMap = new HashMap(); + Map.Entry maxEpochAndNum = new AbstractMap.SimpleImmutableEntry(System.currentTimeMillis(), queueNum); + if (!brokerConfigMap.isEmpty()) { + maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true); + checkIfReusePhysicalQueue(globalIdMap.values()); + checkPhysicalQueueConsistence(brokerConfigMap); + } + if (queueNum < globalIdMap.size()) { + throw new RuntimeException(String.format("Cannot decrease the queue num for static topic %d < %d", queueNum, globalIdMap.size())); + } + //check the queue number + if (queueNum == globalIdMap.size()) { + throw new RuntimeException("The topic queue num is equal the existed queue num, do nothing"); + } + + //the check is ok, now do the mapping allocation + Map brokerNumMap = new HashMap(); + for (String broker: targetBrokers) { + brokerNumMap.put(broker, 0); + } + final Map oldIdToBroker = new HashMap(); + for (Map.Entry entry : globalIdMap.entrySet()) { + String leaderbroker = entry.getValue().getBname(); + oldIdToBroker.put(entry.getKey(), leaderbroker); + if (!brokerNumMap.containsKey(leaderbroker)) { + brokerNumMap.put(leaderbroker, 1); + } else { + brokerNumMap.put(leaderbroker, brokerNumMap.get(leaderbroker) + 1); + } + } + TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(oldIdToBroker, brokerNumMap, null); + allocator.upToNum(queueNum); + Map newIdToBroker = allocator.getIdToBroker(); + + //construct the topic configAndMapping + long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis()); + for (Map.Entry e : newIdToBroker.entrySet()) { + Integer queueId = e.getKey(); + String broker = e.getValue(); + if (globalIdMap.containsKey(queueId)) { + //ignore the exited + continue; + } + TopicConfigAndQueueMapping configMapping; + if (!brokerConfigMap.containsKey(broker)) { + configMapping = new TopicConfigAndQueueMapping(new TopicConfig(topic), new TopicQueueMappingDetail(topic, 0, broker, System.currentTimeMillis())); + configMapping.setWriteQueueNums(1); + configMapping.setReadQueueNums(1); + brokerConfigMap.put(broker, configMapping); + } else { + configMapping = brokerConfigMap.get(broker); + configMapping.setWriteQueueNums(configMapping.getWriteQueueNums() + 1); + configMapping.setReadQueueNums(configMapping.getReadQueueNums() + 1); + } + LogicQueueMappingItem mappingItem = new LogicQueueMappingItem(0, configMapping.getWriteQueueNums() - 1, broker, 0, 0, -1, -1, -1); + TopicQueueMappingDetail.putMappingInfo(configMapping.getMappingDetail(), queueId, new ArrayList(Collections.singletonList(mappingItem))); + } + + // set the topic config + for (Map.Entry entry : brokerConfigMap.entrySet()) { + TopicConfigAndQueueMapping configMapping = entry.getValue(); + configMapping.getMappingDetail().setEpoch(newEpoch); + configMapping.getMappingDetail().setTotalQueues(queueNum); + } + //double check the config + { + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + checkIfReusePhysicalQueue(globalIdMap.values()); + checkPhysicalQueueConsistence(brokerConfigMap); + } + return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_CREATE_OR_UPDATE, newEpoch, brokerConfigMap, new HashSet(), new HashSet()); + } + + + public static TopicRemappingDetailWrapper remappingStaticTopic(String topic, Map brokerConfigMap, Set targetBrokers) { + Map.Entry maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values()); + + //the check is ok, now do the mapping allocation + int maxNum = maxEpochAndNum.getValue(); + + Map brokerNumMap = new HashMap(); + for (String broker: targetBrokers) { + brokerNumMap.put(broker, 0); + } + Map brokerNumMapBeforeRemapping = new HashMap(); + for (TopicQueueMappingOne mappingOne: globalIdMap.values()) { + if (brokerNumMapBeforeRemapping.containsKey(mappingOne.bname)) { + brokerNumMapBeforeRemapping.put(mappingOne.bname, brokerNumMapBeforeRemapping.get(mappingOne.bname) + 1); + } else { + brokerNumMapBeforeRemapping.put(mappingOne.bname, 1); + } + } + + TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap(), brokerNumMap, brokerNumMapBeforeRemapping); + allocator.upToNum(maxNum); + Map expectedBrokerNumMap = allocator.getBrokerNumMap(); + Queue waitAssignQueues = new ArrayDeque(); + //cannot directly use the idBrokerMap from allocator, for the number of globalId maybe not in the natural order + Map expectedIdToBroker = new HashMap(); + //the following logic will make sure that, for one broker, either "map in" or "map out" + //It can't both, map in some queues but also map out some queues. + for (Map.Entry entry : globalIdMap.entrySet()) { + Integer queueId = entry.getKey(); + TopicQueueMappingOne mappingOne = entry.getValue(); + String leaderBroker = mappingOne.getBname(); + if (expectedBrokerNumMap.containsKey(leaderBroker)) { + if (expectedBrokerNumMap.get(leaderBroker) > 0) { + expectedIdToBroker.put(queueId, leaderBroker); + expectedBrokerNumMap.put(leaderBroker, expectedBrokerNumMap.get(leaderBroker) - 1); + } else { + waitAssignQueues.add(queueId); + expectedBrokerNumMap.remove(leaderBroker); + } + } else { + waitAssignQueues.add(queueId); + } + } + + for (Map.Entry entry: expectedBrokerNumMap.entrySet()) { + String broker = entry.getKey(); + Integer queueNum = entry.getValue(); + for (int i = 0; i < queueNum; i++) { + Integer queueId = waitAssignQueues.poll(); + assert queueId != null; + expectedIdToBroker.put(queueId, broker); + } + } + long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis()); + + //Now construct the remapping info + Set brokersToMapOut = new HashSet(); + Set brokersToMapIn = new HashSet(); + for (Map.Entry mapEntry : expectedIdToBroker.entrySet()) { + Integer queueId = mapEntry.getKey(); + String broker = mapEntry.getValue(); + TopicQueueMappingOne topicQueueMappingOne = globalIdMap.get(queueId); + assert topicQueueMappingOne != null; + if (topicQueueMappingOne.getBname().equals(broker)) { + continue; + } + //remapping + final String mapInBroker = broker; + final String mapOutBroker = topicQueueMappingOne.getBname(); + brokersToMapIn.add(mapInBroker); + brokersToMapOut.add(mapOutBroker); + TopicConfigAndQueueMapping mapInConfig = brokerConfigMap.get(mapInBroker); + TopicConfigAndQueueMapping mapOutConfig = brokerConfigMap.get(mapOutBroker); + + if (mapInConfig == null) { + mapInConfig = new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), new TopicQueueMappingDetail(topic, maxNum, mapInBroker, newEpoch)); + brokerConfigMap.put(mapInBroker, mapInConfig); + } + + mapInConfig.setWriteQueueNums(mapInConfig.getWriteQueueNums() + 1); + mapInConfig.setReadQueueNums(mapInConfig.getReadQueueNums() + 1); + + List items = new ArrayList(topicQueueMappingOne.getItems()); + LogicQueueMappingItem last = items.get(items.size() - 1); + items.add(new LogicQueueMappingItem(last.getGen() + 1, mapInConfig.getWriteQueueNums() - 1, mapInBroker, -1, 0, -1, -1, -1)); + + //Use the same object + TopicQueueMappingDetail.putMappingInfo(mapInConfig.getMappingDetail(), queueId, items); + TopicQueueMappingDetail.putMappingInfo(mapOutConfig.getMappingDetail(), queueId, items); + } + + for (Map.Entry entry : brokerConfigMap.entrySet()) { + TopicConfigAndQueueMapping configMapping = entry.getValue(); + configMapping.getMappingDetail().setEpoch(newEpoch); + configMapping.getMappingDetail().setTotalQueues(maxNum); + } + + //double check + { + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values()); + TopicQueueMappingUtils.checkLeaderInTargetBrokers(globalIdMap.values(), targetBrokers); + } + return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_REMAPPING, newEpoch, brokerConfigMap, brokersToMapIn, brokersToMapOut); + } + + public static LogicQueueMappingItem findLogicQueueMappingItem(List mappingItems, long logicOffset, boolean ignoreNegative) { + if (mappingItems == null + || mappingItems.isEmpty()) { + return null; + } + //Could use bi-search to polish performance + for (int i = mappingItems.size() - 1; i >= 0; i--) { + LogicQueueMappingItem item = mappingItems.get(i); + if (ignoreNegative && item.getLogicOffset() < 0) { + continue; + } + if (logicOffset >= item.getLogicOffset()) { + return item; + } + } + //if not found, maybe out of range, return the first one + for (int i = 0; i < mappingItems.size(); i++) { + LogicQueueMappingItem item = mappingItems.get(i); + if (ignoreNegative && item.getLogicOffset() < 0) { + continue; + } + } + return null; + } + + public static LogicQueueMappingItem findNext(List items, LogicQueueMappingItem currentItem, boolean ignoreNegative) { + if (items == null + || currentItem == null) { + return null; + } + for (int i = 0; i < items.size(); i++) { + LogicQueueMappingItem item = items.get(i); + if (ignoreNegative && item.getLogicOffset() < 0) { + continue; + } + if (item.getGen() == currentItem.getGen()) { + if (i < items.size() - 1) { + item = items.get(i + 1); + if (ignoreNegative && item.getLogicOffset() < 0) { + return null; + } else { + return item; + } + } else { + return null; + } + } + } + return null; + } + + + public static boolean checkIfLeader(List items, TopicQueueMappingDetail mappingDetail) { + if (items == null + || mappingDetail == null + || items.isEmpty()) { + return false; + } + return items.get(items.size() - 1).getBname().equals(mappingDetail.getBname()); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicRemappingDetailWrapper.java b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicRemappingDetailWrapper.java new file mode 100644 index 0000000000..ce02a93d9c --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/statictopic/TopicRemappingDetailWrapper.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.common.statictopic; + +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class TopicRemappingDetailWrapper extends RemotingSerializable { + public static final String TYPE_CREATE_OR_UPDATE = "CREATE_OR_UPDATE"; + public static final String TYPE_REMAPPING = "REMAPPING"; + + public static final String SUFFIX_BEFORE = ".before"; + public static final String SUFFIX_AFTER = ".after"; + + + private String topic; + private String type; + private long epoch; + + private Map brokerConfigMap = new HashMap(); + + private Set brokerToMapIn = new HashSet(); + + private Set brokerToMapOut = new HashSet(); + + public TopicRemappingDetailWrapper() { + + } + + public TopicRemappingDetailWrapper(String topic, String type, long epoch, Map brokerConfigMap, Set brokerToMapIn, Set brokerToMapOut) { + this.topic = topic; + this.type = type; + this.epoch = epoch; + this.brokerConfigMap = brokerConfigMap; + this.brokerToMapIn = brokerToMapIn; + this.brokerToMapOut = brokerToMapOut; + } + + public String getTopic() { + return topic; + } + + public String getType() { + return type; + } + + public long getEpoch() { + return epoch; + } + + public Map getBrokerConfigMap() { + return brokerConfigMap; + } + + public Set getBrokerToMapIn() { + return brokerToMapIn; + } + + public Set getBrokerToMapOut() { + return brokerToMapOut; + } + + public void setBrokerConfigMap(Map brokerConfigMap) { + this.brokerConfigMap = brokerConfigMap; + } + + public void setBrokerToMapIn(Set brokerToMapIn) { + this.brokerToMapIn = brokerToMapIn; + } + + public void setBrokerToMapOut(Set brokerToMapOut) { + this.brokerToMapOut = brokerToMapOut; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public void setType(String type) { + this.type = type; + } + + public void setEpoch(long epoch) { + this.epoch = epoch; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java index 9f39f483e6..d534571eb5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java +++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java @@ -25,7 +25,6 @@ public class MessageSysFlag { public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2; public final static int BORNHOST_V6_FLAG = 0x1 << 4; public final static int STOREHOSTADDRESS_V6_FLAG = 0x1 << 5; - public final static int LOGICAL_QUEUE_FLAG = 0x1 << 6; public static int getTransactionValue(final int flag) { return flag & TRANSACTION_ROLLBACK_TYPE; diff --git a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java index ce7558f2bc..20b8ad2081 100644 --- a/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java +++ b/common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java @@ -69,6 +69,10 @@ public class PullSysFlag { return (sysFlag & FLAG_SUSPEND) == FLAG_SUSPEND; } + public static int clearSuspendFlag(final int sysFlag) { + return sysFlag & (~FLAG_SUSPEND); + } + public static boolean hasSubscriptionFlag(final int sysFlag) { return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION; } diff --git a/common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java b/common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java index a884b6a00b..a61ec4c052 100644 --- a/common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java @@ -15,13 +15,10 @@ package org.apache.rocketmq.common;/* * limitations under the License. */ -import org.apache.rocketmq.common.ConfigManager; -import org.apache.rocketmq.common.MixAll; -import org.junit.Test; - import java.io.File; import java.io.PrintWriter; import java.lang.reflect.Method; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; diff --git a/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java index 87a0fc0085..428a9289cd 100644 --- a/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/RegisterBrokerBodyTest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import static org.junit.Assert.assertEquals; import org.junit.Test; @@ -29,7 +30,7 @@ public class RegisterBrokerBodyTest { @Test public void test_encode_decode() throws IOException { RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); - TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper(); registerBrokerBody.setTopicConfigSerializeWrapper(topicConfigSerializeWrapper); ConcurrentMap topicConfigTable = new ConcurrentHashMap(); diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java index 32d2ba6c74..7ef7eaca2a 100644 --- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java @@ -23,6 +23,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Properties; + +import com.alibaba.fastjson.JSON; +import com.google.common.collect.ImmutableList; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; diff --git a/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java b/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java deleted file mode 100644 index 5aa550130d..0000000000 --- a/common/src/test/java/org/apache/rocketmq/common/fastjson/GenericMapSuperclassDeserializerTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.common.fastjson; - -import com.alibaba.fastjson.JSON; -import java.util.Collections; -import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.assertj.core.util.Lists; -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -public class GenericMapSuperclassDeserializerTest { - @Test - public void testLogicalQueuesInfo() throws Exception { - LogicalQueuesInfo logicalQueuesInfo = new LogicalQueuesInfo(); - logicalQueuesInfo.put(0, Lists.newArrayList(new LogicalQueueRouteData(1, 2, new MessageQueue("topic", "broker", 3), MessageQueueRouteState.Normal, 4, 5, 6, 7, "127.1.2.3"))); - - byte[] buf = JSON.toJSONBytes(logicalQueuesInfo); - - LogicalQueuesInfo newLogicalQueuesInfo = JSON.parseObject(buf, LogicalQueuesInfo.class); - - assertThat(newLogicalQueuesInfo).isEqualTo(logicalQueuesInfo); - } - - @Test - public void testLogicalQueuesInfoUnordered() throws Exception { - LogicalQueuesInfoUnordered logicalQueuesInfoUnordered = new LogicalQueuesInfoUnordered(); - MessageQueue mq = new MessageQueue("topic", "broker", 3); - logicalQueuesInfoUnordered.put(0, new ConcurrentHashMap(Collections.singletonMap(new LogicalQueuesInfoUnordered.Key(mq.getBrokerName(), mq.getQueueId(), 4), new LogicalQueueRouteData(1, 2, mq, MessageQueueRouteState.Normal, 4, 5, 6, 7, "127.1.2.3")))); - - byte[] buf = JSON.toJSONBytes(logicalQueuesInfoUnordered); - - LogicalQueuesInfoUnordered newLogicalQueuesInfoUnordered = JSON.parseObject(buf, LogicalQueuesInfoUnordered.class); - - assertThat(newLogicalQueuesInfoUnordered).isEqualTo(logicalQueuesInfoUnordered); - } -} \ No newline at end of file diff --git a/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java new file mode 100644 index 0000000000..f1a2b21c26 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingTest.java @@ -0,0 +1,63 @@ +package org.apache.rocketmq.common.statictopic; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.google.common.collect.ImmutableList; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.theories.suppliers.TestedOn; + +import java.util.Map; + +public class TopicQueueMappingTest { + + @Test + public void testJsonSerialize() { + LogicQueueMappingItem mappingItem = new LogicQueueMappingItem(1, 2, "broker01", 33333333333333333L, 44444444444444444L, 555555555555555555L, 6666666666666666L, 77777777777777777L); + String mappingItemJson = JSON.toJSONString(mappingItem) ; + { + Map mappingItemMap = JSON.parseObject(mappingItemJson, Map.class); + Assert.assertEquals(8, mappingItemMap.size()); + Assert.assertEquals(mappingItemMap.get("bname"), mappingItem.getBname()); + Assert.assertEquals(mappingItemMap.get("gen"), mappingItem.getGen()); + Assert.assertEquals(mappingItemMap.get("logicOffset"), mappingItem.getLogicOffset()); + Assert.assertEquals(mappingItemMap.get("startOffset"), mappingItem.getStartOffset()); + Assert.assertEquals(mappingItemMap.get("endOffset"), mappingItem.getEndOffset()); + Assert.assertEquals(mappingItemMap.get("timeOfStart"), mappingItem.getTimeOfStart()); + Assert.assertEquals(mappingItemMap.get("timeOfEnd"), mappingItem.getTimeOfEnd()); + + } + //test the decode encode + { + LogicQueueMappingItem mappingItemFromJson = RemotingSerializable.fromJson(mappingItemJson, LogicQueueMappingItem.class); + Assert.assertEquals(mappingItem, mappingItemFromJson); + Assert.assertEquals(mappingItemJson, RemotingSerializable.toJson(mappingItemFromJson, false)); + } + TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail("test", 1, "broker01", System.currentTimeMillis()); + TopicQueueMappingDetail.putMappingInfo(mappingDetail, 0, ImmutableList.of(mappingItem)); + + String mappingDetailJson = JSON.toJSONString(mappingDetail); + { + Map mappingDetailMap = JSON.parseObject(mappingDetailJson); + Assert.assertTrue(mappingDetailMap.containsKey("currIdMap")); + Assert.assertEquals(8, mappingDetailMap.size()); + Assert.assertEquals(1, ((JSONObject) mappingDetailMap.get("hostedQueues")).size()); + Assert.assertEquals(1, ((JSONArray)((JSONObject) mappingDetailMap.get("hostedQueues")).get("0")).size()); + } + { + TopicQueueMappingDetail mappingDetailFromJson = RemotingSerializable.decode(mappingDetailJson.getBytes(), TopicQueueMappingDetail.class); + Assert.assertEquals(1, mappingDetailFromJson.getHostedQueues().size()); + Assert.assertEquals(1, mappingDetailFromJson.getHostedQueues().get(0).size()); + Assert.assertEquals(mappingItem, mappingDetailFromJson.getHostedQueues().get(0).get(0)); + Assert.assertEquals(mappingDetailJson, RemotingSerializable.toJson(mappingDetailFromJson, false)); + } + } + + @Test + public void test() { + + } +} diff --git a/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtilsTest.java b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtilsTest.java new file mode 100644 index 0000000000..f5cd0efafd --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtilsTest.java @@ -0,0 +1,303 @@ +package org.apache.rocketmq.common.statictopic; + +import org.apache.rocketmq.common.TopicConfig; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +public class TopicQueueMappingUtilsTest { + + + private Set buildTargetBrokers(int num) { + return buildTargetBrokers(num, ""); + } + + private Set buildTargetBrokers(int num, String suffix) { + Set brokers = new HashSet(); + for (int i = 0; i < num; i++) { + brokers.add("broker" + suffix + i); + } + return brokers; + } + + private Map buildBrokerNumMap(int num) { + Map map = new HashMap(); + for (int i = 0; i < num; i++) { + map.put("broker" + i, 0); + } + return map; + } + + private Map buildBrokerNumMap(int num, int queues) { + Map map = new HashMap(); + int random = new Random().nextInt(num); + for (int i = 0; i < num; i++) { + map.put("broker" + i, queues); + if (i == random) { + map.put("broker" + i, queues + 1); + } + } + return map; + } + + private void testIdToBroker(Map idToBroker, Map brokerNumMap) { + Map brokerNumOther = new HashMap(); + for (int i = 0; i < idToBroker.size(); i++) { + Assert.assertTrue(idToBroker.containsKey(i)); + String broker = idToBroker.get(i); + if (brokerNumOther.containsKey(broker)) { + brokerNumOther.put(broker, brokerNumOther.get(broker) + 1); + } else { + brokerNumOther.put(broker, 1); + } + } + Assert.assertEquals(brokerNumMap.size(), brokerNumOther.size()); + for (Map.Entry entry: brokerNumOther.entrySet()) { + Assert.assertEquals(entry.getValue(), brokerNumMap.get(entry.getKey())); + } + } + + @Test + public void testAllocator() { + //stability + for (int i = 0; i < 10; i++) { + int num = 3; + Map brokerNumMap = buildBrokerNumMap(num); + TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap(), brokerNumMap, null); + allocator.upToNum(num * 2); + for (Map.Entry entry: allocator.getBrokerNumMap().entrySet()) { + Assert.assertEquals(2L, entry.getValue().longValue()); + } + Assert.assertEquals(num * 2, allocator.getIdToBroker().size()); + testIdToBroker(allocator.idToBroker, allocator.getBrokerNumMap()); + + allocator.upToNum(num * 3 - 1); + + for (Map.Entry entry: allocator.getBrokerNumMap().entrySet()) { + Assert.assertTrue(entry.getValue() >= 2); + Assert.assertTrue(entry.getValue() <= 3); + } + Assert.assertEquals(num * 3 - 1, allocator.getIdToBroker().size()); + testIdToBroker(allocator.idToBroker, allocator.getBrokerNumMap()); + } + } + + @Test + public void testRemappingAllocator() { + for (int i = 0; i < 10; i++) { + int num = (i + 2) * 2; + Map brokerNumMap = buildBrokerNumMap(num); + Map brokerNumMapBeforeRemapping = buildBrokerNumMap(num, num); + TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap(), brokerNumMap, brokerNumMapBeforeRemapping); + allocator.upToNum(num * num + 1); + Assert.assertEquals(brokerNumMapBeforeRemapping, allocator.getBrokerNumMap()); + } + } + + + @Test(expected = RuntimeException.class) + public void testTargetBrokersComplete() { + String topic = "static"; + String broker1 = "broker1"; + String broker2 = "broker2"; + Set targetBrokers = new HashSet(); + targetBrokers.add(broker1); + Map brokerConfigMap = new HashMap(); + TopicQueueMappingDetail mappingDetail = new TopicQueueMappingDetail(topic, 0, broker2, 0); + mappingDetail.getHostedQueues().put(1, new ArrayList()); + brokerConfigMap.put(broker2, new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), mappingDetail)); + TopicQueueMappingUtils.checkTargetBrokersComplete(targetBrokers, brokerConfigMap); + } + + + + @Test + public void testCreateStaticTopic() { + String topic = "static"; + int queueNum; + Map brokerConfigMap = new HashMap(); + for (int i = 1; i < 10; i++) { + Set targetBrokers = buildTargetBrokers(2 * i); + Set nonTargetBrokers = buildTargetBrokers(2 * i, "test"); + queueNum = 10 * i; + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap); + Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap); + Assert.assertEquals(2 * i, brokerConfigMap.size()); + + //do the check manually + Map.Entry maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + Assert.assertEquals(queueNum, maxEpochAndNum.getValue().longValue()); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values()); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + + for (Map.Entry entry : brokerConfigMap.entrySet()) { + TopicConfigAndQueueMapping configMapping = entry.getValue(); + if (nonTargetBrokers.contains(configMapping.getMappingDetail().bname)) { + Assert.assertEquals(0, configMapping.getReadQueueNums()); + Assert.assertEquals(0, configMapping.getWriteQueueNums()); + Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size()); + } else { + Assert.assertEquals(5, configMapping.getReadQueueNums()); + Assert.assertEquals(5, configMapping.getWriteQueueNums()); + Assert.assertTrue(configMapping.getMappingDetail().epoch > System.currentTimeMillis()); + for (List items: configMapping.getMappingDetail().getHostedQueues().values()) { + for (LogicQueueMappingItem item: items) { + Assert.assertEquals(0, item.getStartOffset()); + Assert.assertEquals(0, item.getLogicOffset()); + TopicConfig topicConfig = brokerConfigMap.get(item.getBname()); + Assert.assertTrue(item.getQueueId() < topicConfig.getWriteQueueNums()); + } + } + } + } + } + } + + @Test + public void testRemappingStaticTopic() { + String topic = "static"; + int queueNum = 7; + Map brokerConfigMap = new HashMap(); + Set originalBrokers = buildTargetBrokers(2); + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, originalBrokers, brokerConfigMap); + Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap); + Assert.assertEquals(2, brokerConfigMap.size()); + + { + //do the check manually + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values()); + } + + for (int i = 0; i < 10; i++) { + Set targetBrokers = buildTargetBrokers(2, "test" + i); + TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers); + //do the check manually + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values()); + TopicQueueMappingUtils.checkLeaderInTargetBrokers(globalIdMap.values(), targetBrokers); + + Assert.assertEquals((i + 2) * 2, brokerConfigMap.size()); + + //check and complete the logicOffset + for (Map.Entry entry : brokerConfigMap.entrySet()) { + TopicConfigAndQueueMapping configMapping = entry.getValue(); + if (!targetBrokers.contains(configMapping.getMappingDetail().bname)) { + continue; + } + for (List items: configMapping.getMappingDetail().getHostedQueues().values()) { + Assert.assertEquals(i + 2, items.size()); + items.get(items.size() - 1).setLogicOffset(i + 1); + } + } + } + } + + @Test + public void testRemappingStaticTopicStability() { + String topic = "static"; + int queueNum = 7; + Map brokerConfigMap = new HashMap(); + Set originalBrokers = buildTargetBrokers(2); + { + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, originalBrokers, brokerConfigMap); + Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap); + Assert.assertEquals(2, brokerConfigMap.size()); + } + for (int i = 0; i < 10; i++) { + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, originalBrokers); + Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap); + Assert.assertEquals(2, brokerConfigMap.size()); + Assert.assertTrue(wrapper.getBrokerToMapIn().isEmpty()); + Assert.assertTrue(wrapper.getBrokerToMapOut().isEmpty()); + } + } + + + + @Test + public void testUtilsCheck() { + String topic = "static"; + int queueNum = 10; + Map brokerConfigMap = new HashMap(); + Set targetBrokers = buildTargetBrokers(2); + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap); + Assert.assertEquals(wrapper.getBrokerConfigMap(), brokerConfigMap); + Assert.assertEquals(2, brokerConfigMap.size()); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.values().iterator().next(); + List items = configMapping.getMappingDetail().getHostedQueues().values().iterator().next(); + Map.Entry maxEpochNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + int exceptionNum = 0; + try { + configMapping.getMappingDetail().setTopic("xxxx"); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } catch (RuntimeException ignore) { + exceptionNum++; + configMapping.getMappingDetail().setTopic(topic); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } + + try { + configMapping.getMappingDetail().setTotalQueues(1); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } catch (RuntimeException ignore) { + exceptionNum++; + configMapping.getMappingDetail().setTotalQueues(10); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } + + try { + configMapping.getMappingDetail().setEpoch(0); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } catch (RuntimeException ignore) { + exceptionNum++; + configMapping.getMappingDetail().setEpoch(maxEpochNum.getKey()); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } + + + try { + configMapping.getMappingDetail().getHostedQueues().put(10000, new ArrayList(Collections.singletonList(new LogicQueueMappingItem(1, 1, targetBrokers.iterator().next(), 0, 0, -1, -1, -1)))); + TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + } catch (RuntimeException ignore) { + exceptionNum++; + configMapping.getMappingDetail().getHostedQueues().remove(10000); + TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + } + + try { + configMapping.setWriteQueueNums(1); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + } catch (RuntimeException ignore) { + exceptionNum++; + configMapping.setWriteQueueNums(5); + TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap); + } + + try { + items.add(new LogicQueueMappingItem(1, 1, targetBrokers.iterator().next(), 0, 0, -1, -1, -1)); + Map map = TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(map.values()); + } catch (RuntimeException ignore) { + exceptionNum++; + items.remove(items.size() - 1); + Map map = TopicQueueMappingUtils.checkAndBuildMappingItems(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values()), false, true); + TopicQueueMappingUtils.checkIfReusePhysicalQueue(map.values()); + } + Assert.assertEquals(6, exceptionNum); + + } +} diff --git a/docs/cn/statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md b/docs/cn/statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md new file mode 100644 index 0000000000..c06d83f458 --- /dev/null +++ b/docs/cn/statictopic/RocketMQ_Static_Topic_Logic_Queue_设计.md @@ -0,0 +1,498 @@ +### Version 记录 +| 时间 | 主要内容 | 作者 | +| --- | --- | --- | +| 2021-11-01 | 初稿，包括背景、目标、SOT定义与持久化、SOT生命周期、SOT的使用、API逻辑修改、问题与风险 | dongeforever | +| 2021-11-15 | 修改 LogicQueue 的定义，不要引入新字段，完全复用旧的MessageQueue; RemappingStaticTopic时，不要迁移『位点』『幂等数据』等，而是采用Double-Check-Read 的机制| dongforever | +| 2021-12-01 | 更新问题与风险，增加关于一致性、OutOfRange、拉取中断的详细说明| dongforever | +| 2021-12-03 | 增加代码走读的说明| dongforever | +| 2021-12-10 | 引入Scope概念，保留『多集群动态零耦合』的集群设计模型 | dongforever | +| 2021-12-23 | 梳理待完成事项；讨论Admin接口的适配方式 | dongforever | + + + +中文文档在描述特定专业术语时，仍然使用英文。 + +### 需求背景 +StaticTopic/LogicQueue 本质上是解决『固定队列数量』的需求。 +这个需求是不是必需的呢，如果是做应用集成，则可能不是必需的，但如果是做数据集成，则是必需的。 + +固定队列数量，首先可以解决『顺序性』的问题。 +在应用集成场景下，应用是无需感知到队列的，只要MQ能保证按顺序投递给应用即可，MQ底层队列数量如何变化，对应用来说是不关心。比如，MQ之前的那套『禁读禁写』就是可以玩转的。 + +但在数据集成场景中，队列或者叫『分片』，是要暴露给客户端的，客户端所有的数据计算场景，都是基于『分片』来进行的，如果『分片』里的数据，发生了错乱，则计算结果都是错误的。比如，计算WordCount，源数据经过预处理之后，按key写入清洗后的Topic，然后计算侧根据清洗的结果，按照分片来并行计算。如果分片发生变化，则整个清洗逻辑，需要重新处理。 + +有人可能会反驳，说计算组件清洗后，可以以批的方式写入其它存储组件。这当然是可以的，但如果是这样，MQ的价值就纯粹是一个『源头』价值，而不是『通道』价值。 + +MQ要想成为一个『数据通道』，则必需要具备可以让计算组件『回写』数据的能力，具备存储『Clean Data』的能力，这样才让MQ有可能在数据集成领域站稳脚跟。 + +如果是 RocketMQ Streams 这种轻量化的组件，则『回写』会更频繁，更重要。 + +除此之外，『固定队列数据』对于，RocketMQ 自身后续的发展，也是至关重要的： + +- compact topic，如果不能做到严格按key hash，则这个KV系统是有问题的 +- 事务或者其它Coordinator的实现，采用『固定队列数量』，可以选取到准确的Broker来充当协调器 +- Metadata 的存储，按key hash，那么就可以在Broker上，存储千万级的 Topic + +『固定队列数量』对于RocketMQ挺进『数据集成』这个领域，有着不可或缺的作用。 +LogicQueue的思路就是为了解决这一问题。 + +### 设计目标 +#### 总体目标 +提供『Static Topic』的特性。 +引入以下核心概念： +- physical message queue, physical queue for short, a shard bound to a specified broker. +- logic message queue, logic queue for short, a shard vertically composed by physical queues. +- dynamic sharded topic, dynamic topic for short, which has queues increasing with the broker numbers. +- static sharded topic, static topic for short, which has fixed queues, implemented with logic queues. + +『Static Topic』拥有固定的分片数量，每个分片称之为『Logic Queue』。 +每个『Logic Queue』由多个『Physical Queue』进行纵向分段映射组成。 + +引入以下非核心概念，对用户无感知，但对于讨论问题非常重要： +- Leader Queue， 某个『Logic Queue』最新映射的『Physical Queue』，也即可写的那个Queue +- Second Leader Queue，某个『Logic Queue』次新映射的『Physical Queue』，也即最新一次切换之前的『Leader Queue』 + +#### Scope 目标 +单集群固定 和 全网固定, 参考 [The_Scope_Of_Static_Topic](The_Scope_Of_Static_Topic.md)。 + + +#### LogicQueue 目标 +在客户端，LogicQueue 与 Physical Queue 使用体感上没有任何区别，使用一样的概念和对象，遵循一样的语义。 +在服务端，针对 LogicQueue 去适配相关的API。 + +#### 队列语义 +RocketMQ Physical Queue 含有以下语义： + +- 队列内的Offset，单调递增且连续 +- 属于同一个 Broker 上的队列，编号单调递增且连续 + +LogicQueue 需要保障的语义： + +- 队列内的offset，单调递增 + +LogicQueue 可以不保障的语义： + +- 队列内的 offset 连续 +- 属于同一个 Broker 上的队列，编号单调递增且连续 + +offset连续，是一个应该尽量保证的语义，可以允许有少量空洞，但不应该出现大面积不连续的位点。 +offset不连续最直接的问题就是： + +- 计算Lag会比较麻烦 +- 不方便客户端进行各种优化计算（比如切批等） + +但只要空洞不是大量频繁出现的，那么也是问题不大的。 + +单机队列编号连续，除了在注册元数据时，可以简约部分字节外，没有其它实际用处，可以不保证。 +当前客户端使用到『单机队列编号连续』这个特点的场景主要有： + +- 客户端在获取到TopicRouteData后，转化成MessageQueue时，利用编号进行遍历 + + +#### LogicQueue 定义 +当前 MessageQueue 的定义如下 +``` +private String topic; +private String brokerName; +private int queueId; +``` + + +LogicQueue需要对客户直接暴露，为了保证使用习惯一致，采用同样的定义，其中 queueId相当于全局Id，而brokerName 固定如下： +``` +MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME = "__logical_queue_broker__"; +``` +此时，brokerName没有实际含义，但可以用来识别是否是LogicQueue。 + +采用此种定义，对于客户端内部的实现习惯改变如下： + +- **无法直接根据 brokerName 找到addr，而是需要根据 MessageQueue 找到 addr** + +具体改法是MQClientInstance中维护一个映射关系 +``` +private final ConcurrentMap> topicEndPointsTable = new ConcurrentHashMap<>(); +``` + + +基本目标与定义清楚了，接下来的设计，从 Source of Truth 开始。 + +### SOT 定义和持久化 +LogicQueue 的 Source of Truth 就是 LogicQueue 到 Physical Queue 的映射关系。 +只要这个映射关系不丢失，则整个系统的状态都是可以恢复的。 +反之，整个系统可能陷入混乱。 + +这个SOT，命名为 TopicQueueMapping。 + +#### Mapping Schema +``` +{ +"version":"1", +"bname": "broker02" //标记这份数据的原始存储位置，如果发送误拷贝，可以利用这个字段来进行标识 +"epoch": 0, //标记修改版本，用来做一致性校验 +"totalQueues":"50", //当前Topic 总共有多少 LogicQueues +"hostedQueues": { //当前Broker 所拥有的 LogicQueues +"3" : [ + { + "queue":"0", + "bname":"broker01" + "gen":"0", //标记切换代次 + "logicOffset":"0", //logicOffset的起始位置 + "startOffset":"0", // 物理offset的起始位置 + "endOffset":"1000" // 可选，物理offset的最大位置，可以根据上下文算出来 + "timeOfStart":"1561018349243" //可选，用来优化时间搜索 + "timeOfEnd":"1561018349243" //可选，用来优化时间搜索 + "updateTime":"1561018349243" //可选，记录更新的时间 + }, + { + "queue":"0", + "bname":"broker02", + "gen":"1", //可选，标记切换代次 + "logicOffset":"1000", //logicOffset的起始位置 + "startOffset":"0", // 物理offset的起始位置 + "endOffset":"-1" // 可选，物理offset的最大位置，可以根据上下文算出来，最新的一个应该是活跃的 + "timeOfStart":"1561018349243" //可选，用来优化时间搜索 + "timeOfEnd":"1561018349243" //可选，用来优化时间搜索 + "updateTime":"1561018349243" //可选，记录更新的时间 + } + ] + } +} +``` +上述示例的含义是: +* broker02 拥有 LogicQueue 3 +* LogicQueue 3 由 2 个 Physical Queue 组成 +* 位点范围『0-1000』映射到 Physical Queue 『broker01-0』上面 +* 位点范围『1000-』映射到 Physical Queue 『broker02-0』上面 + +『拥有』的定义是指，Leader Queue 在当前Broker。注意，在实现时，也会把Second Leader Queue存储下来作为备份。 + +注意以下要点： + +- 这个数据量会很大，后续需要考虑进行压缩优化（大部分字段可以压缩） +- 如果将来利用 LogicQueue 去做 Serverless 弹缩，则这个数据会加速膨胀，对这个数据的利用要谨慎 +- 要写上 bname，以具备自我识别能力 + +#### Leader Completeness +RocketMQ 没有中心化的元数据存储，那就遵循『Leader Completeness』原则。 +对于每个逻辑队列，把所有映射关系存储在『最新队列所在的Broker上面』，最新队列，其实也是可写队列。 +Leader Completeness，避免了数据的切割，对于后续其它操作有极大的便利。 + +#### Global Epoch Check +对于每个Static Topic，在每个Broker都应该拥有一份『TopicQueueMapping』，每份都带有Epoch。 +在创建和更新时，要对已有数据进行完备性校验，如果发现不完备，则说明上次操作失败，或者部分Broker数据丢失，应该先修复再操作。 + +注意： +即使当前Broker不拥有任何 LogicQueue 或者 PhysicalQueue，也应该存储一份，以做校验。 +假设某个Static Topic只拥有1个Logic Queue，而对应的Broker整好宕机，则此时可以根据其它Broker的信息判断出该Topic不完备。 + + +#### File Isolation +由于 RocketMQ 很多的运维习惯，都是直接拷贝 Topics.json 到别的机器进行部署的。 +而 TopicQueueMapping 是 Broker 相关的，如果把 TopicQueueMapping 从一个Broker拷贝到另一个Broker，则会造成SOT冲突。 + +在设计上，TopicQueueMapping 采取独立文件，避免冲突。 +在格式上，queue 里面要写上 bname，以具备自我识别能力，这样即使误拷贝到另一台机器，可以识别并报错，进行忽略即可。 + + +### SOT 生命周期 +#### 创建和更新 +映射关系的创建，第一期应该只由 MQAdmin 来进行操作。 +后续，可以考虑引入自动化组件。 +这里的要点是： + +- TopicConfig 和 TopicQueueMapping 分开存储，但写入时，需要先写 TopicQueueMapping 再写 TopicConfig（SOT先写） +- 【加强校验】需要在 TopicConfig 里面加上一个字段来标识『LogicQueue』的 Topic +- 【加强校验】允许单独更新 TopicConfig，但要带上 TotalQueues 这些基础数据 +- 允许更新单 LogicQueue + + +更多细节在API逻辑修改里面 +#### 存储 +按照 『Leader Completeness』原则进行存储。 +#### 切换 +如果为了保证严格顺序，则应该采取『禁旧再切新』的原则： +- 从旧 Leader 所在 Broker 获取信息，进行计算 +- 写入旧 Leader，也即禁写旧 Leader +- 写入新 Leader + +如果为了保证最高可用性，则应该采取『切新禁旧再切新』： +- 从旧 Leader 所在 Broker 获取信息，进行计算 +- 写入新 Leader，保证新 Leader 可写，此时 logicOffset 未定 +- 写入旧 Leader，禁写旧 Leader +- 更新新 Leader，确定 logicOffset + +切换失败处理： +- 不管哪种方式，数据存储至少成功了1份，后续可以手工恢复 + + +#### 清除 +有两部分信息需要清除 + +- 旧 Broker 上超过2代的映射关系进行清除 +- 对于单个LogicQueue，清除已经过期的 Broker Queue 映射项目 + + +### SOT 的使用 +#### Broker 注册数据 +SOT存储在Broker上，所以使用从 Broker开始。 + +在 RegisterBrokerBody 中，需要带上两个信息： + +- 对于每个Topic，带上本机队列编号和逻辑队列编号的映射关系，也即queueMap +- 对于每个Topic，需要带上 totalQueueNum 这个信息 + + + +异常情况需要考虑，假如本 Broker 不拥有任何 LogicQueue 呢？依然需要带上 totalQueueNum 这个信息。 +注意，不需要带上所有的映射关系，否则Nameserver很快会被打爆。 + + +#### Nameserver 组装数据 +原先的 QueueData 增加2个字段： + +- totalQueues，标识总逻辑队列数 +- queueMap，也即本机队列编号和逻辑队列编号的映射关系 + + +如果 QueueData 里面 totalQueues 的值 > 0 则认为是逻辑队列，在客户端解析时要进行判断。 + + +遗留问题： +是否需要尊重 readQueueNums 和 writeQueueNums ？ +在LogicQueue这里，这个场景是没有意义的, 但依然保持尊重。 + +#### Client 解析数据 +改动两个方法即可： + +- topicRouteData2TopicPublishInfo +- topicRouteData2TopicSubscribeInfo + + +注意，逻辑队列要求队列数是固定，如果发现，解析完之后，存在部分队列空洞，要用虚拟Broker值进行补全。 +Producer 侧如果要对无 key 场景进行优化，可以通过虚拟Broker值来判断，当前队列是不可用的。 +对于key场景，应该让客户端报错。 + +### API 逻辑修改 +#### Tools +LogicQueue是为了解决『Static Sharding』的问题。对于客户来说，『LogicQueue』是手段，『Static』才是目的。本着『用户知晓目的，开发者才需要关心手段』的原则，对用户应该只暴露『Static』的概念。所有QueueMapping的生命周期维护，应该都对用户透明。 + +#### UpdateStaticTopic +新增UpdateStaticTopic命令，对应RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC=513，主要参数是： + +- -t，topic 名字 +- -qn，总队列数量 +- -c cluster列表 或者 -b broker列表 + + + +UpdateStaticTopic 命令会自动计算预期的分布情况，包括但不限于执行以下逻辑： + +- 检测Topic的命名冲突 +- 检测旧数据的一致性 +- 创建必要的物理队列 +- 创建必要的映射关系 + + +#### RemappingStaticTopic +迁移动作不引入新命令，计算好之后，执行UPDATE_AND_CREATE_STATIC_TOPIC即可. +主要参数： + +- -t， topic 名字 +- -c cluster列表 或者 -b broker列表 + +基本操作流程： + +- 1 获取旧Leader，计算映射关系 +- 2 迁移『位点』『幂等数据』等 +- 3 映射关系写入新/旧 Leader + +其中第二步，数据量可能会很大，导致迁移周期非常长，且不是并发安全的。 +但这些数据，都是覆盖型的，因此可以改造成不迁移的方式，而是在API层做兼容，也即『Double-Read-Check』机制： + +- 读取数据时，先从 Leader 读，如果Leader没有，则从Sub-Leader读取。 +- 提交数据，直接在 Leader 层面操作即可，覆盖旧数据。 + + +将来实现的幂等逻辑，也是类似。 + +#### UpdateTopic +服务端判断『StaticTopic』，禁止该命令进行修改。 + +#### DeleteTopic +复用现有逻辑，对于 StaticTopic，执行必要的清除动作。 + +#### TopicStatus +复用现有逻辑，同时展现『Logical Queue』和『Physical Queue』。 + +#### Broker +#### pullMessage +分段映射，执行远程读，在返回消息时，不进行offset转换，而是返回 OffsetDelta 变量，由客户端进行转换。 +这里的方式，类似于Batch。 + +#### getMinOffset +寻找映射关系，读最早的队列的MinOffset + +#### getMaxOffset +本机读，转换成logicOffset即可。 + + +#### getOffsetByTime +需要分段查找。 +如果要优化查找速度，应该在映射关系里面，插入时间戳。 + + +#### consumerOffsets 系列 +Offset的存储，无需转换，直接存储在 LogicQueue 所对应的最新 PhysicalQueue 中。 +读取时，采取『Double-Read-Check』机制。 + +#### Client + +- MQClientInstance.topicRouteData2TopicXXXInfo，修改解析 TopicRouteData的逻辑 +- Consumer解压消息时，需要加上OffsetDelta逻辑 + +#### SDK 兼容性分析 +如果要使用StaticTopic，则需要升级Client、Broker、Nameserver。 + +### 问题与风险 +#### 数据一致性问题 +RocketMQ 没有引入中心化的存储组件，那么如何保证 SOT 的全局一致性呢？ +主要利用两个信息 +* TopicQueueMapping 带上的 epoch +* TopicQueueMapping 带上 totalQueues + +在更新或者切换时，获取所有Broker上的 TopicQueueMapping，校验 epoch 和 totalQueues，并且根据 TopicQueueMapping 可以完整地构建出对应的Logic Queue，则说明数据是完整一致的。 + +如果发现数据不一致，可能是以下因素引入的： +* 集群中有Broker宕机 +* 上次更新没有完全成功 + +应该要先修复数据，再执行 更新或切换 操作 + +#### No Target Brokers +Target Brokers 是指拥有 LogicQueue 的 Broker。 +考察1个场景，如果某个Topic 只有1个 LogicQueue，而拥有这个 LogicQueue 的 Broker 正好宕机了。此时去更新 Topic，会不会误认为该 Topic 不存在？ +解决这个问题的办法是引入 No Target Brokers，也即集群中除去『Target Brokers』之外的 Broker。 +对于 No Target Broker，依然需要写入一份 TopicQueueMapping，带上 epoch 和 totalQueues，但不拥有任何 LogicQueue。 +有了这个信息之后，在进行一致性校验时，就可以识别出上述场景。 + +尤其要注意，如果 Nameserver 中没有任何信息，则需要主动去所有 Broker 拉取一遍。 + +#### 切换时最新 LogicQueueMappingItem 的 logicOffset 决策问题 +logicOffset的决策，依赖于上一个 PhysicalQueue 的最大位点。 +此时，要么跳跃位点，要么等待上一个 PhysicalQueue 确保已经禁写。 +当前实现，为了保障高可用，采用『切新禁旧再切新』的方式，同时跳跃位点。 + +#### logicOffset 为 -1 时的处理 +此时，可以写，但返回给 客户端的 offset 也是-1。 +此时，不可以读最新 PhysicalQueue。 +需要非常小心触发位点被重置： +- 忽略logicOffset为 -1 的item +- 计算staticOffset时，如果发现logicOffset为-1，则报错 + +目前只允许，SendMessage和GetMin时，返回-1。其余场景，要严格校验并报错。 + + +#### 队列重复映射 +如果允许1个 PhysicalQueue 被重复利用，也即多段映射给多个 LogicQueue，或者从非0开始映射。 +会带来以下麻烦： +* 所有位点相关的API，需要考虑 MappingItem endOffset，因为超过了 endOffset 可能已经不属于 当前 LogicQueue 了 +* 新建 MappingItem，需要先获取 旧 MappingItem 的 endOffset + +当前实现，为了保证简洁，禁止 PhysicalQueue 被重复利用，每次更新映射都会让物理层面的 writeQueues++ 和 readQueues++。 +后续实现，可以考虑复用已经被清除掉的Physical，也即已经没有数据，位点从0开始。 + +#### 备机更新映射 +当前，admin操作都是要求在Master操作的。因此，没有这个问题。 +Command操作时，提前预判Master是否存在，如果不存在，则提前报错，减少中间失败率。 + +#### 拉取消息时 OutOfRange 的判断 +以下情况会影响 OutOfRange 的判断 +* 从备机拉取消息（默认不会返回OFFSET_MOVED） +* 中间 MappingItem 因为Commitlog的提前删除导致 PhysicalQueue Offset Truncation + +所以，OutOfRange 的判断，遵循以下原则： +* 从 Leader Item 拉取，只有requestOffset > maxOffset，尊重 OFFSET_MOVED +* 从 Earliest Item 拉取，只有 requestOffset < minOffset，尊重 OFFSET_MOVED +* 其它情况，都忽略 OFFSET_MOVED + +如果没有恰当地处理 OFFSET_MOVED，可能造成位点被重置。 + +需要注意，这个地方，产生了对 PullMessageResponseHeader 中 minOffset 和 maxOffset 的强依赖。 +在次此之前，这两个信息，只对客户端的限流有作用，对业务没有实际的影响。 + +#### 拉取消息时的 中断问题 +当1个 PhysicalQueue 被拉取干净时，需要修正 nextBeginOffset 到下一个 PhysicalQueue。 +如果没有处理好，则直接会导致拉取中断，无法前进。 +#### pullResult 位点由谁设置的问题 +类似于Batch，由客户端设置，避免服务端解开消息： +在PullResultExt中新增字段 offsetDelta。 +#### Admin接口与User接口的适配方式区别 +User 接口，使用范围广泛如多语言等，应该尽可能简单，把适配逻辑做在服务端，对『客户端』透明。 +那么 Admin 接口呢，比如 examineTopicStats，适配逻辑是做在『服务端』还是『客户端』？ +一个 Admin 接口，通常需要访问所有 Broker 的所有队列。 +如果做在服务端，则可能产生交叉访问，在极端情况下，性能会非常差，举个例子： +100 个 Broker，相互交叉映射过一遍，则Admin客户端首先要向 100 个 Broker 发请求，然后这 100 个 Broker 为了获取远程信息，各自向其余 Broker 发请求。 +其实际网络请求数就是 100 * 100 = 10000 个网络请求。放大效应十分明显。 +同时, 考虑到 Admin 接口，使用范围是有限的，无需考虑多语言适配等问题，可以把适配逻辑做在 Admin 客户端。 + +#### 远程读的性能问题 +从实战经验来看，性能损耗几乎不计。 +#### 使用习惯的改变 +利用新的创建命令进行隔离。 +#### 消费SendBack问题 +目前的实现里，消费Send Back，是直接传回Commitlog Pos，这个在LogicQueue里行不通。 +需要修改API，改成传回『Logic Queue Offset』。 +#### 二阶消息的兼容性 +二阶消息，也即『原始消息』存储在『系统Topic』中，需要经过一轮『Read-ReWrite』逻辑才会被用户看见的消息。 +例如，定时消息，事务消息。 +二阶消息需要支持远程读写操作。 +一期的LogicQueue不支持『二阶消息』。 + +### 待完成事项 +#### 阻止旧客户端的请求 +旧的客户端无法解析逻辑路由，但可以识别物理路由。如果错误使用，则会影响映射关系的准确性。 +#### 阻止Pop模式、事务消息、定时消息使用 LogicQueue +不兼容 事务消息和定时消息。 +LogicQueue 当前不支持Pop模式消费。 +#### ConsumeQueue 的 correctMinOffset 逻辑存在缺陷 +可能导致 LogicQueue 无法清除已经过期的 MappingItem。 +#### getOffsetInQueueByTime 语义有缺陷 +可能导致 LogicQueue 时间搜索不准确。需要专项修复。 +#### Metadata 更新机制 +当前的更新机制太慢。且访问『不存在Broker』时，会频繁访问Nameserver，有打爆Nameserver的风险。 +#### examineConsumeStats 接口获取不到『最近消费时间』 +位点相关的消息可能不在本机，需要远程访问。 +#### resetOffset 需要适配 +当前没有适配。重置位点，可能会得到不符合预期的结果。 +#### MessageQueue 没有被物理清除 +当前只是产生遗留数据，占用一点点存储空间，没有太大影响。 +如果将来要实现 物理 Queue 复用，则需要先完善相关逻辑。 + +### 代码走读要点 +#### Admin 入口 +主要看两个类： +UpdateStaticTopicSubCommand +RemappingStaticTopicSubCommand +#### Metadata 入口 +主要看： +TopicQueueMappingManager +#### Client 入口 +重点关注： +MQClientInstance.updateTopicRouteInfoFromNameServer +#### Server 入口 +以 SendMessageProcessor 为例，插桩代码普遍是以下风格： +``` +TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true); +RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); +if (rewriteResult != null) { + return CompletableFuture.completedFuture(rewriteResult); +} +``` +其它Processor类似 +#### 测试入口 +rocketmq-test模块，statictopic目录。 + + + + + diff --git a/docs/cn/statictopic/The_Scope_Of_Static_Topic.md b/docs/cn/statictopic/The_Scope_Of_Static_Topic.md new file mode 100644 index 0000000000..9c9973c9d6 --- /dev/null +++ b/docs/cn/statictopic/The_Scope_Of_Static_Topic.md @@ -0,0 +1,116 @@ +### Version 记录 +| 时间 | 主要内容 | 作者 | +| --- | --- | --- | +| 2021-11-01 | 初稿，探讨Static Topic的Scope视线范围 | dongeforever | + + +中文文档在描述特定专业术语时，仍然使用英文。 + +### 需求背景 +RocketMQ的集群设计，是一个多集群、动态、零耦合的设计，具体体现在以下地方： +- 一个 Nameserver 可以管理多个 Cluster +- Broker 与 Cluster 之间是弱关联，Cluster仅仅只是一个标识符，主要在运维时使用来界定Topic的创建范围 +- 开发用户对 Cluster 无感知 +- 不同 Broker 之间没有任何关联 + +这样的设计，在运维时带来了极大的便利，但也带来了一个问题: +- Topic 的队列数无法固定 + +基于 Logic Queue 技术而实现的 Static Topic，就是用来解决『固定队列数量』的问题。 + +但这个『固定』要到何种范围呢？是一个值得探讨的问题。 + +从理论上可以分析出来，有以下三种情况： +- 单集群固定 +- 多集群固定 +- 全网固定 + +#### 单集群固定 +一个 Static Topic，固定在一个 Cluster 内漂移。 +不同的 Cluster 内，可以拥有相同的 Static Topic。 +对应MessageQueue的Broker 命名规范为： +``` +__logic__{clusterName} +``` +#### 多集群固定 +一个 Static Topic，固定在特定的几个 Cluster 内漂移。 +没有交集的Cluster集合之间，可以拥有相同的 Static Topic。 +对应MessageQueue的Broker 命名规范为： +``` +__logic__{cluster1}_{cluster2}_{xxx} +``` +#### 全网固定 +全网是指『同一个Nameserver内』。 +一个 Static Topic，不与特定Cluster绑定，同一个Nameserver内，全网漂移。 +同一个Nameserver内，只有一个同名的 Static Topic。 +对应MessageQueue的Broker 命名规范为： +``` +__logic__global +``` +#### 为什么要引入Scope +直接全网固定不就好了吗，为啥还要引入Scope呢？ +主要原因是，不想完全放弃 RocketMQ 『多集群、动态、零耦合』的设计优势。 +而全网固定，则意味着彻底失去了这个优势。 + +举1个『多活保序』的场景: +- ClusterA 部署在 SiteA 内，创建 Static Topic 『TopicTest』，有50个队列。 +- ClusterB 部署在 SiteB 内，创建 Static Topic 『TopicTest』，有50个队列。 + +对Nameserver稍作修改，支持传入标识符(比如为scope或者unitName)，来获取特定范围内的 Topic Route。 + +正常情况下： +- SiteA 的Producer和Consumer 只能看见 ClusterA 的 MessageQueue, brokerName为 "__logic__clusterA"。 +- SiteB 的Producer和Consumer 只能看见 ClusterB 的 MessageQueue, brokerName为 "__logic__clusterB"。 +- 机房内就近访问，且机房内严格保序。 + +假设 SiteA 宕机，此时对Nameserver发指令允许全网读，也即忽略客户端传入的 Scope或者unitName 标识符： +- SiteB 的 Producer 仍然看见并写入 ClusterB 的 MessageQueue, brokerName为 "__logic__clusterB" +- SiteB 的 Consumer 可以同时看见并读取 ClusterA 的 MessageQueue 和 ClusterB MessageQueue, brokerName为 "__logic__clusterB" 和 "__logic__clusterA +- 在这种场景下，Consumer 在消费 ClusterB 数据的同时，同时去消费 ClusterA 未消费完的数据 + +不同地域的客户端，看见不同Scope的元数据，从而访问不同Scope的节点。 + +#### 全球容灾集群 +RocketMQ 多个集群的元数据可以无缝在Nameserver处汇聚，同时又可以无缝地根据标识符拆分给不同地域的Producer和Consumer。 +这样一个『元数据可分可合』的设计优势，是其它消息中间件所不具备的，应该值得挖掘一下。 +引入以下概念： +- 融合集群，共享同一个Nameserver的集群之和 +- 单元集群，clusterName名字一样的集群，不同单元集群之间，物理隔离 +- namespace，租户，逻辑隔离，只是命名的区别 + +如果单元集群部署在异地，所形成的『融合集群』，就是全球容灾集群： +- 客户端引入 scope 或者 unitName 字段，默认情况，不同 scope或者unitName 获取的都是单元集群的元数据 +- 顺序性，关键在于 固定Producer端可见的队列，单元内的队列是固定的，因此可以保证单元内是顺序的 +- Consumer 端按照队列消费，天然是顺序的 +- 正常情况下，单元内封闭，也即单元产生的数据在同单元内消费掉 +- 灾难发生时，改变元数据的可见性，允许读其它 单元集群 未消费完的数据，也即跨单元读 +- 跨单元读，是指读『其它clusterName』的队列，不一定是远程读，如果本单元有相应的Slave节点，则直接本地读 + +### 设计目标 +Static Topic 实现 单集群固定 和 全网固定 两种Scope，以适配『全球容灾集群』。 +多集群，暂时没有必要。 + +一期只实现 全网固定 这个Scope，但在格式上注意兼容 + +#### SOT 增加 Scope 字段 +``` +{ +"version":"1", +"scope": "clusterA", +"bname": "broker02" //标记这份数据的原始存储位置，如果发送误拷贝，可以利用这个字段来进行标识 +"epoch": 0, //标记修改版本，用来做一致性校验 +"totalQueues":"50", //当前Topic 总共有多少 LogicQueues +} +``` + +scope字段： +- 单集群固定，则就是 Cluster 名字 +- 全网固定，则为常量『global』 + + + + + + + + diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java index dd152888aa..a58a3b97fc 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/ClusterTestRequestProcessor.java @@ -56,7 +56,7 @@ public class ClusterTestRequestProcessor extends DefaultRequestProcessor { final GetRouteInfoRequestHeader requestHeader = (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); - TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic(), false); + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); if (topicRouteData != null) { String orderTopicConf = this.namesrvController.getKvConfigManager().getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 85e3883754..98e96dfe7d 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -19,7 +19,6 @@ package org.apache.rocketmq.namesrv.processor; import io.netty.channel.ChannelHandlerContext; import java.io.UnsupportedEncodingException; import java.util.Properties; -import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MQVersion; @@ -53,9 +52,7 @@ import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponse import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; -import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; -import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -353,9 +350,7 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final GetRouteInfoRequestHeader requestHeader = (GetRouteInfoRequestHeader) request.decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); - boolean includeLogicalQueuesInfo = (requestHeader.getSysFlag() & MessageSysFlag.LOGICAL_QUEUE_FLAG) > 0; - - TopicRouteDataNameSrv topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic(), includeLogicalQueuesInfo); + TopicRouteData topicRouteData = this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); if (topicRouteData != null) { if (this.namesrvController.getNamesrvConfig().isOrderMessageEnable()) { @@ -365,16 +360,6 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen topicRouteData.setOrderTopicConf(orderTopicConf); } - Set logicalQueueIdsFilter = requestHeader.getLogicalQueueIdsFilter(); - if (logicalQueueIdsFilter != null) { - LogicalQueuesInfoUnordered logicalQueuesInfo = topicRouteData.getLogicalQueuesInfoUnordered(); - if (logicalQueuesInfo != null) { - LogicalQueuesInfoUnordered filtered = new LogicalQueuesInfoUnordered(logicalQueueIdsFilter.size()); - logicalQueueIdsFilter.forEach(integer -> filtered.put(integer, logicalQueuesInfo.get(integer))); - topicRouteData.setLogicalQueuesInfoUnordered(filtered); - } - } - byte[] content = topicRouteData.encode(); response.setBody(content); response.setCode(ResponseCode.SUCCESS); diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java index ab9e77f3a4..a02d3f1f44 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.namesrv.routeinfo; import io.netty.channel.Channel; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -25,19 +24,18 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; @@ -45,13 +43,10 @@ import org.apache.rocketmq.common.protocol.body.ClusterInfo; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.TopicSysFlag; import org.apache.rocketmq.remoting.common.RemotingUtil; -import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil; public class RouteInfoManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.NAMESRV_LOGGER_NAME); @@ -62,7 +57,8 @@ public class RouteInfoManager { private final HashMap> clusterAddrTable; private final HashMap brokerLiveTable; private final HashMap/* Filter Server */> filterServerTable; - private final ConcurrentMap logicalQueuesInfoTable; + private final HashMap> topicQueueMappingInfoTable; + public RouteInfoManager() { this.topicQueueTable = new HashMap>(1024); @@ -70,7 +66,7 @@ public class RouteInfoManager { this.clusterAddrTable = new HashMap>(32); this.brokerLiveTable = new HashMap(256); this.filterServerTable = new HashMap>(256); - this.logicalQueuesInfoTable = new ConcurrentHashMap<>(1024); + this.topicQueueMappingInfoTable = new HashMap>(1024); } public byte[] getAllClusterInfo() { @@ -158,20 +154,23 @@ public class RouteInfoManager { || registerFirst) { ConcurrentMap tcTable = topicConfigWrapper.getTopicConfigTable(); - Map logicalQueuesInfoMap = topicConfigWrapper.getLogicalQueuesInfoMap(); if (tcTable != null) { for (Map.Entry entry : tcTable.entrySet()) { this.createAndUpdateQueueData(brokerName, entry.getValue()); } } - if (logicalQueuesInfoMap != null) { - long startTime = System.nanoTime(); - for (Map.Entry entry : logicalQueuesInfoMap.entrySet()) { - String topicName = entry.getKey(); - LogicalQueuesInfoUnordered logicalQueuesInfo = ConcurrentHashMapUtil.computeIfAbsent(this.logicalQueuesInfoTable, topicName, ignore -> new LogicalQueuesInfoUnordered()); - mergeLogicalQueuesInfo(brokerName, topicName, logicalQueuesInfo, entry.getValue()); + + TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper = TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper); + + Map topicQueueMappingInfoMap = mappingSerializeWrapper.getTopicQueueMappingInfoMap(); + //the topicQueueMappingInfoMap should never be null, but can be empty + for (Map.Entry entry : topicQueueMappingInfoMap.entrySet()) { + if (!topicQueueMappingInfoTable.containsKey(entry.getKey())) { + topicQueueMappingInfoTable.put(entry.getKey(), new HashMap()); } - log.debug("mergeQueueRouteDataTable topic={} time={}ns", System.nanoTime() - startTime); + //Note asset brokerName equal entry.getValue().getBname() + //here use the mappingDetail.bname + topicQueueMappingInfoTable.get(entry.getKey()).put(entry.getValue().getBname(), entry.getValue()); } } } @@ -403,12 +402,8 @@ public class RouteInfoManager { } } - public TopicRouteDataNameSrv pickupTopicRouteData(final String topic) { - return pickupTopicRouteData(topic, false); - } - - public TopicRouteDataNameSrv pickupTopicRouteData(final String topic, boolean includeLogicalQueuesInfo) { - TopicRouteDataNameSrv topicRouteData = new TopicRouteDataNameSrv(); + public TopicRouteData pickupTopicRouteData(final String topic) { + TopicRouteData topicRouteData = new TopicRouteData(); boolean foundQueueData = false; boolean foundBrokerData = false; Set brokerNameSet = new HashSet(); @@ -417,6 +412,7 @@ public class RouteInfoManager { HashMap> filterServerMap = new HashMap>(); topicRouteData.setFilterServerTable(filterServerMap); + topicRouteData.setTopicQueueMappingByBroker(this.topicQueueMappingInfoTable.get(topic)); try { try { @@ -456,10 +452,6 @@ public class RouteInfoManager { log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); if (foundBrokerData && foundQueueData) { - if (includeLogicalQueuesInfo) { - topicRouteData.setLogicalQueuesInfoUnordered(logicalQueuesInfoTable.get(topic)); - } - return topicRouteData; } @@ -790,34 +782,6 @@ public class RouteInfoManager { return topicList.encode(); } - - private static void mergeLogicalQueuesInfo(String brokerName, String topicName, - LogicalQueuesInfoUnordered logicalQueuesInfoInNamesrv, - LogicalQueuesInfo logicalQueuesInfoFromBroker) { - Set newKeys = logicalQueuesInfoFromBroker.values() - .stream() - .flatMap(Collection::stream) - .filter(v -> Objects.equals(brokerName, v.getBrokerName())) - .map(v -> new LogicalQueuesInfoUnordered.Key(null, v.getQueueId(), v.getOffsetDelta())) - .collect(Collectors.toSet()); - logicalQueuesInfoInNamesrv.values().forEach(m -> - m.values().removeIf(queueRouteData -> - Objects.equals(brokerName, queueRouteData.getBrokerName()) && - !newKeys.contains(new LogicalQueuesInfoUnordered.Key(null, queueRouteData.getQueueId(), queueRouteData.getOffsetDelta())))); - logicalQueuesInfoFromBroker.forEach((logicalQueueId, queueRouteDataListFromBroker) -> { - if (logicalQueueId == null) { - log.warn("queueRouteDataTable topic {} contains null logicalQueueId: {}", topicName, logicalQueuesInfoFromBroker); - return; - } - queueRouteDataListFromBroker.stream() - .filter(queueRouteDataFromBroker -> Objects.equals(brokerName, queueRouteDataFromBroker.getBrokerName())) - .forEach(queueRouteDataFromBroker -> - ConcurrentHashMapUtil.computeIfAbsent(logicalQueuesInfoInNamesrv, logicalQueueId, ignored -> new ConcurrentHashMap<>(queueRouteDataListFromBroker.size())) - .put(new LogicalQueuesInfoUnordered.Key(brokerName, queueRouteDataFromBroker.getQueueId(), queueRouteDataFromBroker.getOffsetDelta()), - queueRouteDataFromBroker) - ); - }); - } } class BrokerLiveInfo { diff --git a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java index 97e6126313..e80dec7ecc 100644 --- a/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java +++ b/namesrv/src/test/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessorTest.java @@ -16,42 +16,27 @@ */ package org.apache.rocketmq.namesrv.processor; -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.DataVersion; -import org.apache.rocketmq.common.MQVersion; -import org.apache.rocketmq.common.MixAll; + import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.common.namesrv.RegisterBrokerResult; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; -import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfoUnordered; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv; -import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.namesrv.routeinfo.RouteInfoManager; @@ -200,7 +185,7 @@ public class DefaultRequestProcessorTest { .contains(new HashMap.SimpleEntry("broker", broker)); } - @Test + /*@Test public void testProcessRequest_RegisterBrokerLogicalQueue() throws Exception { String cluster = "cluster"; String broker1Name = "broker1"; @@ -299,7 +284,7 @@ public class DefaultRequestProcessorTest { assertThat(topicRouteDataNameSrv.getLogicalQueuesInfoUnordered()).isEqualTo(logicalQueuesInfoUnordered); } } - +*/ @Test public void testProcessRequest_RegisterBrokerWithFilterServer() throws RemotingCommandException, NoSuchFieldException, IllegalAccessException { diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index 5ba6cfab0a..158f03f74f 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -161,6 +161,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti } }); + Bootstrap handler = this.bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class) .option(ChannelOption.TCP_NODELAY, true) .option(ChannelOption.SO_KEEPALIVE, false) diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java index 51b619491b..03b16400d0 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java @@ -17,19 +17,23 @@ package org.apache.rocketmq.remoting.protocol; import com.alibaba.fastjson.annotation.JSONField; -import java.lang.annotation.Annotation; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; public class RemotingCommand { public static final String SERIALIZE_TYPE_PROPERTY = "rocketmq.serialize.type"; @@ -93,6 +97,15 @@ public class RemotingCommand { return cmd; } + public static RemotingCommand createResponseCommandWithHeader(int code, CommandCustomHeader customHeader) { + RemotingCommand cmd = new RemotingCommand(); + cmd.setCode(code); + cmd.markResponseType(); + cmd.customHeader = customHeader; + setCmdVersion(cmd); + return cmd; + } + private static void setCmdVersion(RemotingCommand cmd) { if (configVersion >= 0) { cmd.setVersion(configVersion); @@ -110,6 +123,18 @@ public class RemotingCommand { return createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, "not set any response code", classHeader); } + public static RemotingCommand buildErrorResponse(int code, String remark, Class classHeader) { + final RemotingCommand response = RemotingCommand.createResponseCommand(classHeader); + response.setCode(code); + response.setRemark(remark); + return response; + } + + public static RemotingCommand buildErrorResponse(int code, String remark) { + return buildErrorResponse(code, remark, null); + } + + public static RemotingCommand createResponseCommand(int code, String remark, Class classHeader) { RemotingCommand cmd = new RemotingCommand(); @@ -291,11 +316,17 @@ public class RemotingCommand { return objectHeader; } - private Field[] getClazzFields(Class classHeader) { + //make it able to test + Field[] getClazzFields(Class classHeader) { Field[] field = CLASS_HASH_MAP.get(classHeader); if (field == null) { - field = classHeader.getDeclaredFields(); + Set fieldList = new HashSet(); + for (Class className = classHeader; className != Object.class; className = className.getSuperclass()) { + Field[] fields = className.getDeclaredFields(); + fieldList.addAll(Arrays.asList(fields)); + } + field = fieldList.toArray(new Field[0]); synchronized (CLASS_HASH_MAP) { CLASS_HASH_MAP.put(classHeader, field); } diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java index 2bd41cec83..a0fc7655c4 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/RemotingCommandTest.java @@ -19,9 +19,14 @@ package org.apache.rocketmq.remoting.protocol; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.fastjson.JSON; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.junit.Assert; import org.junit.Test; import static org.assertj.core.api.Assertions.assertThat; @@ -198,6 +203,32 @@ public class RemotingCommandTest { Field value = FieldTestClass.class.getDeclaredField("value"); assertThat(method.invoke(remotingCommand, value)).isEqualTo(false); } + + @Test + public void testParentField() throws Exception { + SubExtFieldsHeader subExtFieldsHeader = new SubExtFieldsHeader(); + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(1, subExtFieldsHeader); + Field[] fields = remotingCommand.getClazzFields(subExtFieldsHeader.getClass()); + Set fieldNames = new HashSet<>(); + for (Field field: fields) { + fieldNames.add(field.getName()); + } + Assert.assertTrue(fields.length >= 7); + Set names = new HashSet<>(); + names.add("stringValue"); + names.add("intValue"); + names.add("longValue"); + names.add("booleanValue"); + names.add("doubleValue"); + names.add("name"); + names.add("value"); + for (String name: names) { + Assert.assertTrue(fieldNames.contains(name)); + } + remotingCommand.makeCustomHeaderToNet(); + SubExtFieldsHeader other = (SubExtFieldsHeader) remotingCommand.decodeCommandCustomHeader(subExtFieldsHeader.getClass()); + Assert.assertEquals(other, subExtFieldsHeader); + } } class FieldTestClass { @@ -246,4 +277,72 @@ class ExtFieldsHeader implements CommandCustomHeader { public double getDoubleValue() { return doubleValue; } -} \ No newline at end of file + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ExtFieldsHeader)) return false; + + ExtFieldsHeader that = (ExtFieldsHeader) o; + + if (intValue != that.intValue) return false; + if (longValue != that.longValue) return false; + if (booleanValue != that.booleanValue) return false; + if (Double.compare(that.doubleValue, doubleValue) != 0) return false; + return stringValue != null ? stringValue.equals(that.stringValue) : that.stringValue == null; + } + + @Override + public int hashCode() { + int result; + long temp; + result = stringValue != null ? stringValue.hashCode() : 0; + result = 31 * result + intValue; + result = 31 * result + (int) (longValue ^ (longValue >>> 32)); + result = 31 * result + (booleanValue ? 1 : 0); + temp = Double.doubleToLongBits(doubleValue); + result = 31 * result + (int) (temp ^ (temp >>> 32)); + return result; + } +} + + +class SubExtFieldsHeader extends ExtFieldsHeader { + private String name = "12321"; + private int value = 111; + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof SubExtFieldsHeader)) return false; + if (!super.equals(o)) return false; + + SubExtFieldsHeader that = (SubExtFieldsHeader) o; + + if (value != that.value) return false; + return name != null ? name.equals(that.name) : that.name == null; + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + (name != null ? name.hashCode() : 0); + result = 31 * result + value; + return result; + } +} diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalConsumer.java index ce739be591..71f9088875 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalConsumer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalConsumer.java @@ -49,6 +49,7 @@ public class RMQNormalConsumer extends AbstractMQConsumer { consumer = new DefaultMQPushConsumer(consumerGroup); consumer.setInstanceName(RandomUtil.getStringByUUID()); consumer.setNamesrvAddr(nsAddr); + consumer.setPollNameServerInterval(100); try { consumer.subscribe(topic, subExpression); } catch (MQClientException e) { @@ -92,4 +93,8 @@ public class RMQNormalConsumer extends AbstractMQConsumer { create(); start(); } + + public DefaultMQPushConsumer getConsumer() { + return consumer; + } } diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java index 6bbec6cb94..001db9584e 100644 --- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQNormalProducer.java @@ -73,6 +73,7 @@ public class RMQNormalProducer extends AbstractMQProducer { producer.setProducerGroup(getProducerGroupName()); producer.setInstanceName(getProducerInstanceName()); producer.setUseTLS(useTLS); + producer.setPollNameServerInterval(100); if (nsAddr != null) { producer.setNamesrvAddr(nsAddr); @@ -132,6 +133,12 @@ public class RMQNormalProducer extends AbstractMQProducer { } } + public void send(int num, MessageQueue mq) { + for (int i = 0; i < num; i++) { + sendMQ((Message) getMessageByTag(null), mq); + } + } + public ResultWrapper sendMQ(Message msg, MessageQueue mq) { org.apache.rocketmq.client.producer.SendResult metaqResult = null; try { diff --git a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java index df6abfc662..1e0a19a2b3 100644 --- a/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java +++ b/test/src/main/java/org/apache/rocketmq/test/clientinterface/AbstractMQProducer.java @@ -34,6 +34,7 @@ public abstract class AbstractMQProducer extends MQCollector implements MQProduc protected String producerInstanceName = null; protected boolean isDebug = false; + public AbstractMQProducer(String topic) { super(); producerGroupName = RandomUtil.getStringByUUID(); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java deleted file mode 100644 index 8c0caa6a27..0000000000 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdmin.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.test.util; - -import java.util.HashMap; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ForkJoinPool; -import org.apache.log4j.Logger; -import org.apache.rocketmq.common.admin.TopicStatsTable; -import org.apache.rocketmq.common.protocol.body.ClusterInfo; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; - -public class MQAdmin { - private static Logger log = Logger.getLogger(MQAdmin.class); - - public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, - int queueNum) { - int defaultWaitTime = 5; - return createTopic(nameSrvAddr, clusterName, topic, queueNum, defaultWaitTime); - } - - public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, - int queueNum, int waitTimeSec) { - boolean createResult = false; - DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); - mqAdminExt.setInstanceName(UUID.randomUUID().toString()); - mqAdminExt.setNamesrvAddr(nameSrvAddr); - try { - mqAdminExt.start(); - mqAdminExt.createTopic(clusterName, topic, queueNum); - } catch (Exception e) { - } - - long startTime = System.currentTimeMillis(); - while (!createResult) { - createResult = checkTopicExist(mqAdminExt, topic); - if (System.currentTimeMillis() - startTime < waitTimeSec * 1000) { - TestUtils.waitForMoment(100); - } else { - log.error(String.format("timeout,but create topic[%s] failed!", topic)); - break; - } - } - - ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); - return createResult; - } - - private static boolean checkTopicExist(DefaultMQAdminExt mqAdminExt, String topic) { - boolean createResult = false; - try { - TopicStatsTable topicInfo = mqAdminExt.examineTopicStats(topic); - createResult = !topicInfo.getOffsetTable().isEmpty(); - } catch (Exception e) { - } - - return createResult; - } - - public static boolean createSub(String nameSrvAddr, String clusterName, String consumerId) { - boolean createResult = true; - DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); - mqAdminExt.setNamesrvAddr(nameSrvAddr); - SubscriptionGroupConfig config = new SubscriptionGroupConfig(); - config.setGroupName(consumerId); - try { - mqAdminExt.start(); - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(mqAdminExt, - clusterName); - for (String addr : masterSet) { - try { - mqAdminExt.createAndUpdateSubscriptionGroupConfig(addr, config); - log.info(String.format("create subscription group %s to %s success.\n", consumerId, - addr)); - } catch (Exception e) { - e.printStackTrace(); - Thread.sleep(1000 * 1); - } - } - } catch (Exception e) { - createResult = false; - e.printStackTrace(); - } - ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); - return createResult; - } - - public static ClusterInfo getCluster(String nameSrvAddr) { - DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); - mqAdminExt.setNamesrvAddr(nameSrvAddr); - ClusterInfo clusterInfo = null; - try { - mqAdminExt.start(); - clusterInfo = mqAdminExt.examineBrokerClusterInfo(); - } catch (Exception e) { - e.printStackTrace(); - } - ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); - return clusterInfo; - } - - public static boolean isBrokerExist(String ns, String ip) { - ClusterInfo clusterInfo = getCluster(ns); - if (clusterInfo == null) { - return false; - } else { - HashMap brokers = clusterInfo.getBrokerAddrTable(); - for (String brokerName : brokers.keySet()) { - HashMap brokerIps = brokers.get(brokerName).getBrokerAddrs(); - for (long brokerId : brokerIps.keySet()) { - if (brokerIps.get(brokerId).contains(ip)) - return true; - } - } - } - - return false; - } - - public void getSubConnection(String nameSrvAddr, String clusterName, String consumerId) { - boolean createResult = true; - DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); - mqAdminExt.setNamesrvAddr(nameSrvAddr); - SubscriptionGroupConfig config = new SubscriptionGroupConfig(); - config.setGroupName(consumerId); - try { - mqAdminExt.start(); - Set masterSet = CommandUtil.fetchMasterAddrByClusterName(mqAdminExt, - clusterName); - for (String addr : masterSet) { - try { - - System.out.printf("create subscription group %s to %s success.\n", consumerId, - addr); - } catch (Exception e) { - e.printStackTrace(); - Thread.sleep(1000 * 1); - } - } - } catch (Exception e) { - createResult = false; - e.printStackTrace(); - } - ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); - } - -} diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java new file mode 100644 index 0000000000..8287d810ce --- /dev/null +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.util; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.log4j.Logger; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.common.statictopic.TopicRemappingDetailWrapper; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminUtils; +import org.apache.rocketmq.tools.command.CommandUtil; +import org.apache.rocketmq.tools.command.topic.RemappingStaticTopicSubCommand; +import org.apache.rocketmq.tools.command.topic.UpdateStaticTopicSubCommand; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ForkJoinPool; + +public class MQAdminTestUtils { + private static Logger log = Logger.getLogger(MQAdminTestUtils.class); + + public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, + int queueNum) { + int defaultWaitTime = 5; + return createTopic(nameSrvAddr, clusterName, topic, queueNum, defaultWaitTime); + } + + public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, + int queueNum, int waitTimeSec) { + boolean createResult = false; + DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); + mqAdminExt.setInstanceName(UUID.randomUUID().toString()); + mqAdminExt.setNamesrvAddr(nameSrvAddr); + try { + mqAdminExt.start(); + mqAdminExt.createTopic(clusterName, topic, queueNum); + } catch (Exception e) { + } + + long startTime = System.currentTimeMillis(); + while (!createResult) { + createResult = checkTopicExist(mqAdminExt, topic); + if (System.currentTimeMillis() - startTime < waitTimeSec * 1000) { + TestUtils.waitForMoment(100); + } else { + log.error(String.format("timeout,but create topic[%s] failed!", topic)); + break; + } + } + + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); + return createResult; + } + + private static boolean checkTopicExist(DefaultMQAdminExt mqAdminExt, String topic) { + boolean createResult = false; + try { + TopicStatsTable topicInfo = mqAdminExt.examineTopicStats(topic); + createResult = !topicInfo.getOffsetTable().isEmpty(); + } catch (Exception e) { + } + + return createResult; + } + + public static boolean createSub(String nameSrvAddr, String clusterName, String consumerId) { + boolean createResult = true; + DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); + mqAdminExt.setNamesrvAddr(nameSrvAddr); + SubscriptionGroupConfig config = new SubscriptionGroupConfig(); + config.setGroupName(consumerId); + try { + mqAdminExt.start(); + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(mqAdminExt, + clusterName); + for (String addr : masterSet) { + try { + mqAdminExt.createAndUpdateSubscriptionGroupConfig(addr, config); + log.info(String.format("create subscription group %s to %s success.\n", consumerId, + addr)); + } catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000 * 1); + } + } + } catch (Exception e) { + createResult = false; + e.printStackTrace(); + } + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); + return createResult; + } + + public static ClusterInfo getCluster(String nameSrvAddr) { + DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); + mqAdminExt.setNamesrvAddr(nameSrvAddr); + ClusterInfo clusterInfo = null; + try { + mqAdminExt.start(); + clusterInfo = mqAdminExt.examineBrokerClusterInfo(); + } catch (Exception e) { + e.printStackTrace(); + } + ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); + return clusterInfo; + } + + public static boolean isBrokerExist(String ns, String ip) { + ClusterInfo clusterInfo = getCluster(ns); + if (clusterInfo == null) { + return false; + } else { + HashMap brokers = clusterInfo.getBrokerAddrTable(); + for (String brokerName : brokers.keySet()) { + HashMap brokerIps = brokers.get(brokerName).getBrokerAddrs(); + for (long brokerId : brokerIps.keySet()) { + if (brokerIps.get(brokerId).contains(ip)) + return true; + } + } + } + + return false; + } + + //should only be test, if some middle operation failed, it dose not backup the brokerConfigMap + public static Map createStaticTopic(String topic, int queueNum, Set targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception { + Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + assert brokerConfigMap.isEmpty(); + TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap); + MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt); + MQAdminUtils.updateTopicConfigMappingAll(brokerConfigMap, defaultMQAdminExt, false); + return brokerConfigMap; + } + + //should only be test, if some middle operation failed, it dose not backup the brokerConfigMap + public static void remappingStaticTopic(String topic, Set targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception { + Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + assert !brokerConfigMap.isEmpty(); + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers); + MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt); + MQAdminUtils.remappingStaticTopic(topic, wrapper.getBrokerToMapIn(), wrapper.getBrokerToMapOut(), brokerConfigMap, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, false, defaultMQAdminExt); + } + + //for test only + public static void remappingStaticTopicWithNegativeLogicOffset(String topic, Set targetBrokers, DefaultMQAdminExt defaultMQAdminExt) throws Exception { + Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + assert !brokerConfigMap.isEmpty(); + TopicRemappingDetailWrapper wrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers); + MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt); + remappingStaticTopicWithNegativeLogicOffset(topic, wrapper.getBrokerToMapIn(), wrapper.getBrokerToMapOut(), brokerConfigMap, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, false, defaultMQAdminExt); + } + + //for test only + public static void remappingStaticTopicWithNegativeLogicOffset(String topic, Set brokersToMapIn, Set brokersToMapOut, Map brokerConfigMap, int blockSeqSize, boolean force, DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + + ClientMetadata clientMetadata = MQAdminUtils.getBrokerMetadata(defaultMQAdminExt); + MQAdminUtils.checkIfMasterAlive(brokerConfigMap.keySet(), defaultMQAdminExt, clientMetadata); + // now do the remapping + //Step1: let the new leader can be write without the logicOffset + for (String broker : brokersToMapIn) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + //Step2: forbid the write of old leader + for (String broker : brokersToMapOut) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + + //Step5: write the non-target brokers + for (String broker : brokerConfigMap.keySet()) { + if (brokersToMapIn.contains(broker) || brokersToMapOut.contains(broker)) { + continue; + } + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + } + + public static void createStaticTopicWithCommand(String topic, int queueNum, Set brokers, String cluster, String nameservers) throws Exception { + UpdateStaticTopicSubCommand cmd = new UpdateStaticTopicSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] args; + if (cluster != null) { + args = new String[]{ + "-c", cluster, + "-t", topic, + "-qn", String.valueOf(queueNum), + "-n", nameservers + }; + } else { + String brokerStr = String.join(",", brokers); + args = new String[]{ + "-b", brokerStr, + "-t", topic, + "-qn", String.valueOf(queueNum), + "-n", nameservers + }; + } + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), args, cmd.buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + return; + } + if (commandLine.hasOption('n')) { + String namesrvAddr = commandLine.getOptionValue('n'); + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr); + } + cmd.execute(commandLine, options, null); + } + + public static void remappingStaticTopicWithCommand(String topic, Set brokers, String cluster, String nameservers) throws Exception { + RemappingStaticTopicSubCommand cmd = new RemappingStaticTopicSubCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] args; + if (cluster != null) { + args = new String[]{ + "-c", cluster, + "-t", topic, + "-n", nameservers + }; + } else { + String brokerStr = String.join(",", brokers); + args = new String[]{ + "-b", brokerStr, + "-t", topic, + "-n", nameservers + }; + } + final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), args, cmd.buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + return; + } + if (commandLine.hasOption('n')) { + String namesrvAddr = commandLine.getOptionValue('n'); + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr); + } + cmd.execute(commandLine, options, null); + } + +} diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index 79469e1f3b..a92389ea37 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -19,8 +19,10 @@ package org.apache.rocketmq.test.base; import com.google.common.collect.ImmutableList; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -29,7 +31,6 @@ import org.apache.log4j.Logger; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.client.consumer.MQPullConsumer; import org.apache.rocketmq.client.consumer.MQPushConsumer; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.MQProducer; import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.common.MQVersion; @@ -44,7 +45,7 @@ import org.apache.rocketmq.test.clientinterface.AbstractMQConsumer; import org.apache.rocketmq.test.clientinterface.AbstractMQProducer; import org.apache.rocketmq.test.factory.ConsumerFactory; import org.apache.rocketmq.test.listener.AbstractListener; -import org.apache.rocketmq.test.util.MQAdmin; +import org.apache.rocketmq.test.util.MQAdminTestUtils; import org.apache.rocketmq.test.util.MQRandomUtils; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.admin.MQAdminExt; @@ -55,6 +56,8 @@ public class BaseConf { public final static String nsAddr; protected final static String broker1Name; protected final static String broker2Name; + //the logic queue test need at least three brokers + protected final static String broker3Name; protected final static String clusterName; protected final static int brokerNum; protected final static int waitTime = 5; @@ -63,6 +66,7 @@ public class BaseConf { protected final static NamesrvController namesrvController; protected final static BrokerController brokerController1; protected final static BrokerController brokerController2; + protected final static BrokerController brokerController3; protected final static List brokerControllerList; protected final static Map brokerControllerMap; protected final static List mqClients = new ArrayList(); @@ -75,11 +79,13 @@ public class BaseConf { nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort(); brokerController1 = IntegrationTestBase.createAndStartBroker(nsAddr); brokerController2 = IntegrationTestBase.createAndStartBroker(nsAddr); + brokerController3 = IntegrationTestBase.createAndStartBroker(nsAddr); clusterName = brokerController1.getBrokerConfig().getBrokerClusterName(); broker1Name = brokerController1.getBrokerConfig().getBrokerName(); broker2Name = brokerController2.getBrokerConfig().getBrokerName(); - brokerNum = 2; - brokerControllerList = ImmutableList.of(brokerController1, brokerController2); + broker3Name = brokerController3.getBrokerConfig().getBrokerName(); + brokerNum = 3; + brokerControllerList = ImmutableList.of(brokerController1, brokerController2, brokerController3); brokerControllerMap = brokerControllerList.stream().collect(Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity())); } @@ -97,7 +103,10 @@ public class BaseConf { List brokerDatas = mqAdminExt.examineTopicRouteInfo(clusterName).getBrokerDatas(); return brokerDatas.size() == brokerNum; }); - } catch (MQClientException e) { + for (BrokerController brokerController: brokerControllerList) { + brokerController.getBrokerOuterAPI().refreshMetadata(); + } + } catch (Exception e) { log.error("init failed, please check BaseConf"); } ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); @@ -116,10 +125,19 @@ public class BaseConf { } public static String initConsumerGroup(String group) { - MQAdmin.createSub(nsAddr, clusterName, group); + MQAdminTestUtils.createSub(nsAddr, clusterName, group); return group; } + public static DefaultMQAdminExt getAdmin(String nsAddr) { + final DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(500); + mqAdminExt.setNamesrvAddr(nsAddr); + mqAdminExt.setPollNameServerInterval(100); + mqClients.add(mqAdminExt); + return mqAdminExt; + } + + public static RMQNormalProducer getProducer(String nsAddr, String topic) { return getProducer(nsAddr, topic, false); } @@ -197,6 +215,14 @@ public class BaseConf { shutdown(mqClients); } + public static Set getBrokers() { + Set brokers = new HashSet<>(); + brokers.add(broker1Name); + brokers.add(broker2Name); + brokers.add(broker3Name); + return brokers; + } + public static void shutdown(List mqClients) { mqClients.forEach(mqClient -> ForkJoinPool.commonPool().execute(() -> { if (mqClient instanceof AbstractMQProducer) { diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 50dc8fc64a..56ee320dba 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -34,7 +34,7 @@ import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.apache.rocketmq.test.util.MQAdmin; +import org.apache.rocketmq.test.util.MQAdminTestUtils; import org.apache.rocketmq.test.util.TestUtils; public class IntegrationTestBase { @@ -129,7 +129,7 @@ public class IntegrationTestBase { String baseDir = createBaseDir(); BrokerConfig brokerConfig = new BrokerConfig(); MessageStoreConfig storeConfig = new MessageStoreConfig(); - brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.getAndIncrement()); + brokerConfig.setBrokerName(BROKER_NAME_PREFIX + BROKER_INDEX.incrementAndGet()); brokerConfig.setBrokerIP1("127.0.0.1"); brokerConfig.setNamesrvAddr(nsAddr); brokerConfig.setEnablePropertyFilter(true); @@ -139,6 +139,7 @@ public class IntegrationTestBase { storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE); storeConfig.setMaxIndexNum(INDEX_NUM); storeConfig.setMaxHashSlotNum(INDEX_NUM * 4); + storeConfig.setDeleteWhen("01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23;00"); return createAndStartBroker(storeConfig, brokerConfig); } @@ -166,7 +167,7 @@ public class IntegrationTestBase { boolean createResult; while (true) { - createResult = MQAdmin.createTopic(nsAddr, clusterName, topic, queueNumbers); + createResult = MQAdminTestUtils.createTopic(nsAddr, clusterName, topic, queueNumbers); if (createResult) { break; } else if (System.currentTimeMillis() - startTime > topicCreateTime) { diff --git a/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java b/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java deleted file mode 100644 index b1e9013f67..0000000000 --- a/test/src/test/java/org/apache/rocketmq/test/smoke/LogicalQueueIT.java +++ /dev/null @@ -1,1170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.test.smoke; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Maps; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.client.consumer.PullCallback; -import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.client.consumer.PullStatus; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.SendResultForLogicalQueue; -import org.apache.rocketmq.common.MQVersion; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.ResponseCode; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.namesrv.NamesrvController; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.store.CommitLog; -import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.MappedFileQueue; -import org.apache.rocketmq.test.base.BaseConf; -import org.apache.rocketmq.test.base.IntegrationTestBase; -import org.apache.rocketmq.test.util.MQRandomUtils; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExtImpl; -import org.apache.rocketmq.tools.command.logicalqueue.MigrateTopicLogicalQueueCommand; -import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; -import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueNumCommand; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Optional.ofNullable; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.awaitility.Awaitility.await; -import static org.awaitility.Awaitility.waitAtMost; - -@FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class LogicalQueueIT { - private static final Logger logger = LoggerFactory.getLogger(LogicalQueueIT.class); - public static String nsAddr; - private static String broker1Name; - private static String broker2Name; - private static String clusterName; - private static int brokerNum; - private final static int QUEUE_NUMBERS = 8; - private static NamesrvController namesrvController; - private static BrokerController brokerController1; - private static BrokerController brokerController2; - private static Map brokerControllerMap; - private final static List mqClients = new ArrayList<>(); - - private static DefaultMQProducer producer; - private static DefaultMQPullConsumer consumer; - private static DefaultMQAdminExt mqAdminExt; - private static volatile String topic = null; - private static final String placeholderTopic = "placeholder"; - private static final int MSG_SENT_TIMES = 3; - private static final int COMMIT_LOG_FILE_SIZE = 512 * 1024; - - @BeforeClass - public static void beforeClass() throws Exception { - System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION)); - namesrvController = IntegrationTestBase.createAndStartNamesrv(); - nsAddr = "127.0.0.1:" + namesrvController.getNettyServerConfig().getListenPort(); - - int oldCommitLogSize = IntegrationTestBase.COMMIT_LOG_SIZE; - IntegrationTestBase.COMMIT_LOG_SIZE = COMMIT_LOG_FILE_SIZE; - brokerController1 = IntegrationTestBase.createAndStartBroker(nsAddr); - brokerController2 = IntegrationTestBase.createAndStartBroker(nsAddr); - IntegrationTestBase.COMMIT_LOG_SIZE = oldCommitLogSize; - - clusterName = brokerController1.getBrokerConfig().getBrokerClusterName(); - broker1Name = brokerController1.getBrokerConfig().getBrokerName(); - broker2Name = brokerController2.getBrokerConfig().getBrokerName(); - brokerNum = 2; - brokerControllerMap = ImmutableList.of(brokerController1, brokerController2).stream().collect(Collectors.toMap(input -> input.getBrokerConfig().getBrokerName(), Function.identity())); - - BaseConf.waitBrokerRegistered(nsAddr, clusterName); - - producer = new DefaultMQProducer(MQRandomUtils.getRandomConsumerGroup()); - mqClients.add(producer); - producer.setNamesrvAddr(nsAddr); - producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); - producer.setSendMsgTimeout(1000); - producer.start(); - - consumer = new DefaultMQPullConsumer(BaseConf.initConsumerGroup()); - mqClients.add(consumer); - consumer.setNamesrvAddr(nsAddr); - consumer.setConsumerPullTimeoutMillis(1000); - consumer.start(); - - mqAdminExt = new DefaultMQAdminExt(1000); - mqClients.add(mqAdminExt); - mqAdminExt.setNamesrvAddr(nsAddr); - mqAdminExt.start(); - - mqAdminExt.createTopic(clusterName, placeholderTopic, 1); - } - - @AfterClass - public static void afterClass() { - BaseConf.shutdown(mqClients); - brokerControllerMap.forEach((s, brokerController) -> brokerController.shutdown()); - ofNullable(namesrvController).ifPresent(obj -> ForkJoinPool.commonPool().execute(obj::shutdown)); - } - - @Before - public void setUp() throws Exception { - topic = "tt-" + MQRandomUtils.getRandomTopic(); - logger.info("use topic: {}", topic); - mqAdminExt.createTopic(clusterName, topic, QUEUE_NUMBERS); - assertThat(mqAdminExt.examineTopicRouteInfo(topic).getBrokerDatas()).hasSize(brokerNum); - await().atMost(5, TimeUnit.SECONDS).until(() -> !mqAdminExt.examineTopicStats(topic).getOffsetTable().isEmpty()); - - consumer.setRegisterTopics(Collections.singleton(topic)); - // consumer.setMessageQueueListener & consumer.registerMessageQueueListener are useless in DefaultMQPullConsumer, they will never work, so do not need to test it - - new UpdateTopicLogicalQueueMappingCommand().execute(mqAdminExt, topic, brokerControllerMap.values().stream().map(BrokerController::getBrokerAddr).collect(Collectors.toSet())); - } - - private static String getCurrentMethodName() { - // 0: getStackTrace - // 1: getCurrentMethodName - // 2: __realMethod__ - return Thread.currentThread().getStackTrace()[2].getMethodName(); - } - - @Test - public void test001_SendPullSync() throws Exception { - String methodName = getCurrentMethodName(); - - List publishMessageQueues = producer.fetchPublishMessageQueues(topic); - assertThat(publishMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); - Set queueIds = IntStream.range(0, brokerNum * QUEUE_NUMBERS).boxed().collect(Collectors.toSet()); - for (MessageQueue messageQueue : publishMessageQueues) { - assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(queueIds.remove(messageQueue.getQueueId())).isTrue(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); - } - } - assertThat(queueIds).isEmpty(); - - List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); - subscribeMessageQueues.sort(Comparator.comparingInt(MessageQueue::getQueueId)); - queueIds.addAll(IntStream.range(0, brokerNum * QUEUE_NUMBERS).boxed().collect(Collectors.toSet())); - for (MessageQueue messageQueue : subscribeMessageQueues) { - assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(queueIds.remove(messageQueue.getQueueId())).isTrue(); - long offset = mqAdminExt.minOffset(messageQueue); - PullResult pullResult = consumer.pull(messageQueue, "*", offset, 10); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); - offset = -1; - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = pullResult.getMsgFoundList().get(i); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, messageQueue.getQueueId(), i)); - if (i > 0) { - assertThat(msg.getQueueOffset()).isEqualTo(offset + i); - } else { - offset = msg.getQueueOffset(); - } - } - assertThat(maxOffsetUncommitted(messageQueue)).isEqualTo(offset + MSG_SENT_TIMES); - } - assertThat(queueIds).isEmpty(); - } - - @Test - public void test002_SendPullAsync() throws Exception { - String methodName = getCurrentMethodName(); - - List publishMessageQueues = producer.fetchPublishMessageQueues(topic); - for (MessageQueue messageQueue : publishMessageQueues) { - for (int i = 0; i < MSG_SENT_TIMES; i++) { - CompletableFuture future = new CompletableFuture<>(); - producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue, new SendCallback() { - @Override public void onSuccess(SendResult sendResult) { - future.complete(sendResult); - } - - @Override public void onException(Throwable e) { - future.completeExceptionally(e); - } - }); - SendResult sendResult = future.get(); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); - } - } - - List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - for (MessageQueue messageQueue : subscribeMessageQueues) { - long offset = mqAdminExt.minOffset(messageQueue); - CompletableFuture future = new CompletableFuture<>(); - consumer.pull(messageQueue, "*", offset, 10, new PullCallback() { - @Override public void onSuccess(PullResult pullResult) { - future.complete(pullResult); - } - - @Override public void onException(Throwable e) { - future.completeExceptionally(e); - } - }); - PullResult pullResult = future.get(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); - offset = -1; - Iterator it = pullResult.getMsgFoundList().iterator(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = it.next(); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, messageQueue.getQueueId(), i)); - if (i > 0) { - assertThat(msg.getQueueOffset()).isEqualTo(offset + i); - } else { - offset = msg.getQueueOffset(); - } - } - } - } - - @Test - public void test003_MigrateOnceWithoutData() throws Exception { - final String methodName = getCurrentMethodName(); - - final int logicalQueueIdx = 1; - - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - List logicalQueueRouteDataList1 = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - LogicalQueueRouteData lastLogicalQueueRouteData1 = logicalQueueRouteDataList1.get(logicalQueueRouteDataList1.size() - 1); - String newBrokerName; - if (lastLogicalQueueRouteData1.getBrokerName().equals(broker1Name)) { - newBrokerName = broker2Name; - } else { - newBrokerName = broker1Name; - } - - MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); - - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, null); - - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - assertThat(topicRouteInfo.getLogicalQueuesInfo()).isNotNull(); - for (Map.Entry> entry : topicRouteInfo.getLogicalQueuesInfo().entrySet()) { - List logicalQueueRouteDataList2 = entry.getValue(); - if (entry.getKey() == logicalQueueIdx) { - assertThat(logicalQueueRouteDataList2).hasSize(logicalQueueRouteDataList1.size() + 1); - LogicalQueueRouteData lastLogicalQueueRouteData2 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 2); - assertThat(lastLogicalQueueRouteData2.getMessageQueue()).isEqualTo(lastLogicalQueueRouteData1.getMessageQueue()); - assertThat(lastLogicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); - assertThat(lastLogicalQueueRouteData2.getMessagesCount()).isEqualTo(0L); - assertThat(lastLogicalQueueRouteData2.isWritable()).isFalse(); - assertThat(lastLogicalQueueRouteData2.isReadable()).isFalse(); - assertThat(lastLogicalQueueRouteData2.isExpired()).isTrue(); - assertThat(lastLogicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(0L); - - LogicalQueueRouteData lastLogicalQueueRouteData3 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 1); - assertThat(lastLogicalQueueRouteData3.getBrokerName()).isEqualTo(newBrokerName); - assertThat(lastLogicalQueueRouteData3.getOffsetMax()).isLessThan(0L); - assertThat(lastLogicalQueueRouteData3.isWritable()).isTrue(); - assertThat(lastLogicalQueueRouteData3.isReadable()).isTrue(); - assertThat(lastLogicalQueueRouteData3.isExpired()).isFalse(); - assertThat(lastLogicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(0L); - } else { - assertThat(logicalQueueRouteDataList2).hasSize(1); - LogicalQueueRouteData logicalQueueRouteData = logicalQueueRouteDataList2.get(0); - assertThat(logicalQueueRouteData.getOffsetMax()).isLessThan(0L); - assertThat(logicalQueueRouteData.isWritable()).isTrue(); - assertThat(logicalQueueRouteData.isReadable()).isTrue(); - assertThat(logicalQueueRouteData.isExpired()).isFalse(); - assertThat(logicalQueueRouteData.getLogicalQueueDelta()).isEqualTo(0L); - } - } - - List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); - for (MessageQueue mq : subscribeMessageQueues) { - assertThat(mqAdminExt.minOffset(mq)).isEqualTo(0L); - } - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); - } - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - CompletableFuture future = new CompletableFuture<>(); - producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue, new SendCallback() { - @Override public void onSuccess(SendResult sendResult) { - future.complete(sendResult); - } - - @Override public void onException(Throwable e) { - future.completeExceptionally(e); - } - }); - SendResult sendResult = future.get(); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); - } - - assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(2 * MSG_SENT_TIMES); - - waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == 2 * MSG_SENT_TIMES); - - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", 0L, 2 * MSG_SENT_TIMES); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMinOffset()).isEqualTo(0); - assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); - List msgFoundList = pullResult.getMsgFoundList(); - assertThat(msgFoundList).hasSize(2 * MSG_SENT_TIMES); - Iterator it = pullResult.getMsgFoundList().iterator(); - long offset = 0L; - for (String prefix : new String[] {"sync", "async"}) { - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = it.next(); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%s-%d-%d", methodName, prefix, migratedMessageQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset); - offset++; - } - } - - offset = pullResult.getNextBeginOffset(); - CompletableFuture future = new CompletableFuture<>(); - consumer.pull(migratedMessageQueue, "*", offset, 10, new PullCallback() { - @Override public void onSuccess(PullResult pullResult) { - future.complete(pullResult); - } - - @Override public void onException(Throwable e) { - future.completeExceptionally(e); - } - }); - pullResult = future.get(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); - assertThat(pullResult.getMinOffset()).isEqualTo(0); - assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getMsgFoundList()).isNull(); - } - - @Test - public void test004_MigrateOnceWithData() throws Exception { - final String methodName = getCurrentMethodName(); - - final int logicalQueueIdx = 1; - - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - List logicalQueueRouteDataList1 = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - LogicalQueueRouteData lastLogicalQueueRouteData1 = logicalQueueRouteDataList1.get(logicalQueueRouteDataList1.size() - 1); - String newBrokerName; - if (lastLogicalQueueRouteData1.getBrokerName().equals(broker1Name)) { - newBrokerName = broker2Name; - } else { - newBrokerName = broker1Name; - } - - MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - } - assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(MSG_SENT_TIMES); - - waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == MSG_SENT_TIMES); - - { - long offset = 0L; - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMinOffset()).isEqualTo(0); - assertThat(pullResult.getMaxOffset()).isEqualTo(MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(MSG_SENT_TIMES); - List msgFoundList = pullResult.getMsgFoundList(); - assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); - Iterator it = pullResult.getMsgFoundList().iterator(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = it.next(); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset); - offset++; - } - } - - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, null); - - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - assertThat(topicRouteInfo.getLogicalQueuesInfo()).isNotNull(); - for (Map.Entry> entry : topicRouteInfo.getLogicalQueuesInfo().entrySet()) { - List logicalQueueRouteDataList2 = entry.getValue(); - if (entry.getKey() == logicalQueueIdx) { - assertThat(logicalQueueRouteDataList2).hasSize(logicalQueueRouteDataList1.size() + 1); - LogicalQueueRouteData lastLogicalQueueRouteData2 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 2); - assertThat(lastLogicalQueueRouteData2.getMessageQueue()).isEqualTo(lastLogicalQueueRouteData1.getMessageQueue()); - assertThat(lastLogicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); - assertThat(lastLogicalQueueRouteData2.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); - assertThat(lastLogicalQueueRouteData2.isWritable()).isFalse(); - assertThat(lastLogicalQueueRouteData2.isReadable()).isTrue(); - assertThat(lastLogicalQueueRouteData2.isExpired()).isFalse(); - assertThat(lastLogicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(0L); - - LogicalQueueRouteData lastLogicalQueueRouteData3 = logicalQueueRouteDataList2.get(logicalQueueRouteDataList2.size() - 1); - assertThat(lastLogicalQueueRouteData3.getBrokerName()).isEqualTo(newBrokerName); - assertThat(lastLogicalQueueRouteData3.getOffsetMax()).isLessThan(0L); - assertThat(lastLogicalQueueRouteData3.isWritable()).isTrue(); - assertThat(lastLogicalQueueRouteData3.isReadable()).isTrue(); - assertThat(lastLogicalQueueRouteData3.isExpired()).isFalse(); - assertThat(lastLogicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(MSG_SENT_TIMES); - } else { - assertThat(logicalQueueRouteDataList2).hasSize(1); - LogicalQueueRouteData logicalQueueRouteData = logicalQueueRouteDataList2.get(0); - assertThat(logicalQueueRouteData.getOffsetMax()).isLessThan(0L); - assertThat(logicalQueueRouteData.isWritable()).isTrue(); - assertThat(logicalQueueRouteData.isReadable()).isTrue(); - assertThat(logicalQueueRouteData.isExpired()).isFalse(); - assertThat(logicalQueueRouteData.getLogicalQueueDelta()).isEqualTo(0L); - } - } - assertThat(migratedMessageQueue).isNotNull(); - - List subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - assertThat(subscribeMessageQueues).hasSize(brokerNum * QUEUE_NUMBERS); - for (MessageQueue mq : subscribeMessageQueues) { - assertThat(mqAdminExt.minOffset(mq)).isEqualTo(0L); - } - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - CompletableFuture future = new CompletableFuture<>(); - producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue, new SendCallback() { - @Override public void onSuccess(SendResult sendResult) { - future.complete(sendResult); - } - - @Override public void onException(Throwable e) { - future.completeExceptionally(e); - } - }); - SendResult sendResult = future.get(); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migratedMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(newBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); - } - - assertThat(maxOffsetUncommitted(migratedMessageQueue)).isEqualTo(2 * MSG_SENT_TIMES); - - waitAtMost(5, TimeUnit.SECONDS).until(() -> mqAdminExt.maxOffset(migratedMessageQueue) == 2 * MSG_SENT_TIMES); - - long offset = 0L; - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMinOffset()).isEqualTo(0); - assertThat(pullResult.getMaxOffset()).isEqualTo(MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(MSG_SENT_TIMES); - List msgFoundList = pullResult.getMsgFoundList(); - assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); - Iterator it = pullResult.getMsgFoundList().iterator(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = it.next(); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-sync-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset); - offset++; - } - - offset = pullResult.getNextBeginOffset(); - CompletableFuture pullResultFuture = new CompletableFuture<>(); - consumer.pull(migratedMessageQueue, "*", offset, 2 * MSG_SENT_TIMES, new PullCallback() { - @Override public void onSuccess(PullResult pullResult) { - pullResultFuture.complete(pullResult); - } - - @Override public void onException(Throwable e) { - pullResultFuture.completeExceptionally(e); - } - }); - pullResult = pullResultFuture.get(); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMinOffset()).isEqualTo(MSG_SENT_TIMES); - assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); - msgFoundList = pullResult.getMsgFoundList(); - assertThat(msgFoundList).hasSize(MSG_SENT_TIMES); - it = pullResult.getMsgFoundList().iterator(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = it.next(); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(migratedMessageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-async-%d-%d", methodName, migratedMessageQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset); - offset++; - } - - offset = pullResult.getNextBeginOffset(); - pullResult = consumer.pull(migratedMessageQueue, "*", offset, 10); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); - assertThat(pullResult.getMinOffset()).isEqualTo(MSG_SENT_TIMES); - assertThat(pullResult.getMaxOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getNextBeginOffset()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(pullResult.getMsgFoundList()).isNull(); - } - - @Test - public void test005_MigrateWithDataBackAndForth() throws Exception { - final String methodName = getCurrentMethodName(); - - final int logicalQueueIdx = 1; - - MessageQueue migratedMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); - - BrokerController brokerController; - - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - LogicalQueueRouteData lastLogicalQueueRouteData; - { - List logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - lastLogicalQueueRouteData = logicalQueueRouteDataList.get(logicalQueueRouteDataList.size() - 1); - } - final String fromBrokerName, toBrokerName, fromBrokerAddr, toBrokerAddr; - if (lastLogicalQueueRouteData.getBrokerName().equals(broker1Name)) { - fromBrokerName = broker1Name; - fromBrokerAddr = brokerController1.getBrokerAddr(); - toBrokerName = broker2Name; - toBrokerAddr = brokerController2.getBrokerAddr(); - } else { - fromBrokerName = broker2Name; - fromBrokerAddr = brokerController2.getBrokerAddr(); - toBrokerName = broker1Name; - toBrokerAddr = brokerController1.getBrokerAddr(); - } - - int msgIdx = 0; - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(fromBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(logicalQueueIdx); - } - - rotateBrokerCommitLog(brokerControllerMap.get(fromBrokerName)); - - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, toBrokerName, null); - - { - LogicalQueuesInfo info; - List logicalQueueRouteDataList; - info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(2); - info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(1); - } - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(toBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); - } - - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, fromBrokerName, null); - // now will reuse queue with a ReadOnly one - - { - LogicalQueuesInfo info; - List logicalQueueRouteDataList; - info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(3); - info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(2); - } - - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(fromBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(logicalQueueIdx); - } - - LogicalQueueRouteData logicalQueueRouteData1; - LogicalQueueRouteData logicalQueueRouteData2; - { - List logicalQueueRouteDataList; - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(3); - logicalQueueRouteData1 = logicalQueueRouteDataList.get(0); - assertThat(logicalQueueRouteData1.getLogicalQueueDelta()).isEqualTo(0); - assertThat(logicalQueueRouteData1.isReadable()).isTrue(); - assertThat(logicalQueueRouteData1.isWritable()).isFalse(); - assertThat(logicalQueueRouteData1.isExpired()).isFalse(); - assertThat(logicalQueueRouteData1.isWriteOnly()).isFalse(); - assertThat(logicalQueueRouteData1.getBrokerName()).isEqualTo(fromBrokerName); - assertThat(logicalQueueRouteData1.getOffsetMax()).isGreaterThanOrEqualTo(0L); - assertThat(logicalQueueRouteData1.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); - assertThat(logicalQueueRouteData1.getFirstMsgTimeMillis()).isGreaterThan(0L); - assertThat(logicalQueueRouteData1.getLastMsgTimeMillis()).isGreaterThan(0L); - logicalQueueRouteData2 = logicalQueueRouteDataList.get(1); - assertThat(logicalQueueRouteData2.getLogicalQueueDelta()).isEqualTo(MSG_SENT_TIMES); - assertThat(logicalQueueRouteData2.isReadable()).isTrue(); - assertThat(logicalQueueRouteData2.isWritable()).isFalse(); - assertThat(logicalQueueRouteData2.isExpired()).isFalse(); - assertThat(logicalQueueRouteData2.isWriteOnly()).isFalse(); - assertThat(logicalQueueRouteData2.getBrokerName()).isEqualTo(toBrokerName); - assertThat(logicalQueueRouteData2.getOffsetMax()).isGreaterThanOrEqualTo(0L); - assertThat(logicalQueueRouteData2.getMessagesCount()).isEqualTo(MSG_SENT_TIMES); - assertThat(logicalQueueRouteData2.getFirstMsgTimeMillis()).isGreaterThan(0L); - assertThat(logicalQueueRouteData2.getLastMsgTimeMillis()).isGreaterThan(0L); - LogicalQueueRouteData logicalQueueRouteData3 = logicalQueueRouteDataList.get(2); - assertThat(logicalQueueRouteData3.getLogicalQueueDelta()).isEqualTo(2 * MSG_SENT_TIMES); - assertThat(logicalQueueRouteData3.isReadable()).isTrue(); - assertThat(logicalQueueRouteData3.isWritable()).isTrue(); - assertThat(logicalQueueRouteData3.isExpired()).isFalse(); - assertThat(logicalQueueRouteData3.isWriteOnly()).isFalse(); - assertThat(logicalQueueRouteData3.getBrokerName()).isEqualTo(fromBrokerName); - assertThat(logicalQueueRouteData3.getOffsetMax()).isLessThan(0L); - } - - msgIdx = 0; - forLoop: - for (long offset = 0L; ; ) { - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); - switch (pullResult.getPullStatus()) { - case NO_NEW_MSG: - assertThat(offset).isGreaterThanOrEqualTo(3L * MSG_SENT_TIMES); - break forLoop; - case OFFSET_ILLEGAL: - offset = pullResult.getNextBeginOffset(); - break; - default: - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMsgFoundList()).isNotNull(); - assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); - for (MessageExt msg : pullResult.getMsgFoundList()) { - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx)); - msgIdx++; - assertThat(msg.getQueueOffset()).isEqualTo(offset); - offset++; - } - offset = pullResult.getNextBeginOffset(); - break; - } - } - - waitAtMost(5, TimeUnit.SECONDS).until(() -> maxOffsetUncommitted(logicalQueueRouteData1.getMessageQueue()) == mqAdminExt.maxOffset(logicalQueueRouteData1.getMessageQueue())); - waitAtMost(5, TimeUnit.SECONDS).until(() -> maxOffsetUncommitted(logicalQueueRouteData2.getMessageQueue()) == mqAdminExt.maxOffset(logicalQueueRouteData2.getMessageQueue())); - - // now verify after commit log cleaned, toBroker's first queue route data will be expired too - brokerController = brokerControllerMap.get(logicalQueueRouteData2.getBrokerName()); - rotateBrokerCommitLog(brokerController); - deleteCommitLogFiles(brokerController, 1); - - { - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - List logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).hasSize(2); - assertThat(logicalQueueRouteDataList.get(0)).isEqualToIgnoringGivenFields(new LogicalQueueRouteData(logicalQueueIdx, 0, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.ReadOnly, 0, 3, -1, -1, fromBrokerAddr), "firstMsgTimeMillis", "lastMsgTimeMillis"); - assertThat(logicalQueueRouteDataList.get(1)).isEqualToComparingFieldByField(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr)); - } - - // try pull again, since there is an expired queue route in the middle. - { - int msgCount = 0; - Queue wantMsgIdx = new LinkedList<>(); - wantMsgIdx.addAll(IntStream.range(0, MSG_SENT_TIMES).boxed().collect(Collectors.toList())); - wantMsgIdx.addAll(IntStream.range(2 * MSG_SENT_TIMES, 3 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); - forLoop: - for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); - switch (pullResult.getPullStatus()) { - case NO_NEW_MSG: - assertThat(msgCount).as("offset=%d", offset).isEqualTo(2 * MSG_SENT_TIMES); - break forLoop; - case OFFSET_ILLEGAL: - offset = pullResult.getNextBeginOffset(); - break; - case FOUND: - msgCount += pullResult.getMsgFoundList().size(); - boolean first = true; - for (MessageExt msg : pullResult.getMsgFoundList()) { - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); - if (first) { - assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); - first = false; - } else { - assertThat(msg.getQueueOffset()).isGreaterThan(offset); - } - offset = msg.getQueueOffset(); - } - offset = pullResult.getNextBeginOffset(); - break; - default: - Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); - } - } - } - - // rotate first queue route to expired, and pull it - brokerController = brokerControllerMap.get(logicalQueueRouteData1.getBrokerName()); - rotateBrokerCommitLog(brokerController); - deleteCommitLogFiles(brokerController, 2); - - { - List logicalQueueRouteDataList; - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr))); - } - - { - int msgCount = 0; - Queue wantMsgIdx = new LinkedList<>(); - wantMsgIdx.addAll(IntStream.range(2 * MSG_SENT_TIMES, 3 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); - forLoop: - for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); - switch (pullResult.getPullStatus()) { - case NO_NEW_MSG: - if (msgCount != MSG_SENT_TIMES) { - Assert.fail(String.format(Locale.ENGLISH, "want %d msg but got %d", MSG_SENT_TIMES, msgCount)); - } - break forLoop; - case OFFSET_ILLEGAL: - offset = pullResult.getNextBeginOffset(); - break; - case FOUND: - msgCount += pullResult.getMsgFoundList().size(); - boolean first = true; - for (MessageExt msg : pullResult.getMsgFoundList()) { - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); - if (first) { - assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); - first = false; - } else { - assertThat(msg.getQueueOffset()).isGreaterThan(offset); - } - offset = msg.getQueueOffset(); - } - offset = pullResult.getNextBeginOffset(); - break; - default: - Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); - } - } - } - - brokerController = brokerControllerMap.get(fromBrokerName); - rotateBrokerCommitLog(brokerController); - deleteCommitLogFiles(brokerController, 1); - - { - forLoop: - for (long offset = mqAdminExt.minOffset(migratedMessageQueue); ; ) { - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); - // commit log rotate and cleaned, so there is no message. - switch (pullResult.getPullStatus()) { - case NO_MATCHED_MSG: - case NO_NEW_MSG: - assertThat(pullResult.getNextBeginOffset()).isEqualTo(3 * MSG_SENT_TIMES); - break forLoop; - case OFFSET_ILLEGAL: - offset = pullResult.getNextBeginOffset(); - break; - default: - Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); - } - } - } - - { - LogicalQueuesInfo logicalQueuesInfo = mqAdminExt.queryTopicLogicalQueueMapping(brokerController.getBrokerAddr(), topic); - List logicalQueueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, fromBrokerAddr))); - } - - // try migrate to this broker which has a expired queue, expect it will reuse the expired one, pull it to verify if delta works well - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, toBrokerName, null); - - { - List logicalQueueRouteDataList; - topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - logicalQueueRouteDataList = topicRouteInfo.getLogicalQueuesInfo().get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).isEqualTo(Arrays.asList( - new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Expired, MSG_SENT_TIMES, 2 * MSG_SENT_TIMES, 0, 0, fromBrokerAddr) - , new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr) - )); - - LogicalQueuesInfo info; - info = mqAdminExt.queryTopicLogicalQueueMapping(fromBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).isEqualTo(Arrays.asList( - new LogicalQueueRouteData(logicalQueueIdx, 2 * MSG_SENT_TIMES, new MessageQueue(topic, fromBrokerName, logicalQueueIdx), MessageQueueRouteState.Expired, MSG_SENT_TIMES, 2 * MSG_SENT_TIMES, 0, 0, fromBrokerAddr) - , new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr) - )); - info = mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic); - logicalQueueRouteDataList = info.get(logicalQueueIdx); - assertThat(logicalQueueRouteDataList).isEqualTo(Collections.singletonList(new LogicalQueueRouteData(logicalQueueIdx, 3 * MSG_SENT_TIMES, new MessageQueue(topic, toBrokerName, QUEUE_NUMBERS), MessageQueueRouteState.Normal, MSG_SENT_TIMES, -1, -1, -1, toBrokerAddr))); - } - - msgIdx = 3 * MSG_SENT_TIMES; - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, msgIdx++).getBytes(StandardCharsets.UTF_8)), migratedMessageQueue); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(toBrokerName); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(QUEUE_NUMBERS); - } - - { - int msgCount = 0; - Queue wantMsgIdx = new LinkedList<>(); - wantMsgIdx.addAll(IntStream.range(3 * MSG_SENT_TIMES, 4 * MSG_SENT_TIMES).boxed().collect(Collectors.toList())); - LOOP: - for (long offset = 0L; ; ) { - PullResult pullResult = consumer.pull(migratedMessageQueue, "*", offset, 3 * MSG_SENT_TIMES); - switch (pullResult.getPullStatus()) { - case NO_NEW_MSG: - assertThat(msgCount).as("msgCount with offset=%d", offset).isEqualTo(MSG_SENT_TIMES); - break LOOP; - case OFFSET_ILLEGAL: - assertThat(pullResult.getNextBeginOffset()).isNotEqualTo(Long.MIN_VALUE); - offset = pullResult.getNextBeginOffset(); - break; - case FOUND: - msgCount += pullResult.getMsgFoundList().size(); - boolean first = true; - for (MessageExt msg : pullResult.getMsgFoundList()) { - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).as("offset=%d", offset).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, logicalQueueIdx, wantMsgIdx.poll())); - if (first) { - assertThat(msg.getQueueOffset()).isGreaterThanOrEqualTo(offset); - first = false; - } else { - assertThat(msg.getQueueOffset()).isGreaterThan(offset); - } - offset = msg.getQueueOffset(); - } - offset = pullResult.getNextBeginOffset(); - break; - default: - Assert.fail(String.format(Locale.ENGLISH, "unexpected pull offset=%d status: %s", offset, pullResult)); - } - } - } - } - - @Test - public void test006_LogicalQueueNumChanged() throws Exception { - String methodName = getCurrentMethodName(); - int logicalQueueNum = brokerNum * QUEUE_NUMBERS; - - List publishMessageQueues; - publishMessageQueues = producer.fetchPublishMessageQueues(topic); - assertThat(publishMessageQueues).hasSize(logicalQueueNum); - List subscribeMessageQueues; - subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - assertThat(subscribeMessageQueues).hasSize(logicalQueueNum); - - logicalQueueNum++; - new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); - - int newAddLogicalQueueIdx = logicalQueueNum - 1; - MessageQueue newAddLogicalQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, newAddLogicalQueueIdx); - String newAddLogicalQueueBrokerName; - { - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - LogicalQueuesInfo info = topicRouteInfo.getLogicalQueuesInfo(); - assertThat(info).isNotNull(); - List queueRouteDataList = info.get(newAddLogicalQueueIdx); - assertThat(queueRouteDataList).isNotNull(); - assertThat(queueRouteDataList).hasSize(1); - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(0); - newAddLogicalQueueBrokerName = queueRouteData.getBrokerName(); - assertThat(queueRouteData.getState()).isEqualTo(MessageQueueRouteState.Normal); - assertThat(queueRouteData.getLogicalQueueDelta()).isEqualTo(0); - assertThat(queueRouteData.getLogicalQueueIndex()).isEqualTo(newAddLogicalQueueIdx); - } - - publishMessageQueues = producer.fetchPublishMessageQueues(topic); - assertThat(publishMessageQueues).hasSize(logicalQueueNum); - Set logicalQueueIds = IntStream.range(0, logicalQueueNum).boxed().collect(Collectors.toSet()); - Map> queueIds = Maps.newHashMap(); - for (String brokerName : Arrays.asList(broker1Name, broker2Name)) { - queueIds.put(brokerName, IntStream.range(0, QUEUE_NUMBERS).boxed().collect(Collectors.toSet())); - } - queueIds.get(newAddLogicalQueueBrokerName).add(QUEUE_NUMBERS); - for (MessageQueue messageQueue : publishMessageQueues) { - assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(logicalQueueIds.remove(messageQueue.getQueueId())).isTrue(); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, messageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), messageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(messageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(messageQueue.getQueueId()); - if (i == 0) { - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(queueIds.get(sendResult2.getOrigBrokerName()).remove(sendResult2.getOrigQueueId())).as("brokerName %s queueId %d", sendResult2.getOrigBrokerName(), sendResult2.getOrigQueueId()).isTrue(); - } - } - } - assertThat(logicalQueueIds).isEmpty(); - - subscribeMessageQueues = consumer.fetchSubscribeMessageQueues(topic).stream().sorted().collect(Collectors.toList()); - assertThat(subscribeMessageQueues).hasSize(logicalQueueNum); - subscribeMessageQueues.sort(Comparator.comparingInt(MessageQueue::getQueueId)); - logicalQueueIds.addAll(IntStream.range(0, logicalQueueNum).boxed().collect(Collectors.toSet())); - for (MessageQueue messageQueue : subscribeMessageQueues) { - assertThat(messageQueue.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(logicalQueueIds.remove(messageQueue.getQueueId())).isTrue(); - long offset = mqAdminExt.minOffset(messageQueue); - assertThat(offset).isEqualTo(0); - PullResult pullResult = consumer.pull(messageQueue, "*", offset, 10); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = pullResult.getMsgFoundList().get(i); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(messageQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, messageQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset + i); - } - assertThat(maxOffsetUncommitted(messageQueue)).isEqualTo(offset + MSG_SENT_TIMES); - } - assertThat(logicalQueueIds).isEmpty(); - - // increase TopicConfig write queue first then increase logical queue, expect to reuse - String broker2Addr = brokerController2.getBrokerAddr(); - TopicConfig topicConfig = mqAdminExt.examineTopicConfig(broker2Addr, topic); - topicConfig.setWriteQueueNums(topicConfig.getWriteQueueNums() + 1); - topicConfig.setReadQueueNums(topicConfig.getReadQueueNums() + 1); - mqAdminExt.createAndUpdateTopicConfig(broker2Addr, topicConfig); - logicalQueueNum++; - new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); - { - newAddLogicalQueueIdx = logicalQueueNum -1; - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - LogicalQueuesInfo info = topicRouteInfo.getLogicalQueuesInfo(); - assertThat(info).isNotNull(); - List queueRouteDataList = info.get(newAddLogicalQueueIdx); - assertThat(queueRouteDataList).isNotNull(); - assertThat(queueRouteDataList).hasSize(1); - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(0); - assertThat(queueRouteData.getState()).isEqualTo(MessageQueueRouteState.Normal); - assertThat(queueRouteData.getLogicalQueueDelta()).isEqualTo(0); - assertThat(queueRouteData.getLogicalQueueIndex()).isEqualTo(newAddLogicalQueueIdx); - assertThat(queueRouteData.getBrokerName()).isEqualTo(broker2Name); - assertThat(queueRouteData.getQueueId()).isEqualTo(topicConfig.getWriteQueueNums() -1); - } - - logicalQueueNum-=2; - new UpdateTopicLogicalQueueNumCommand().execute(mqAdminExt, clusterName, topic, logicalQueueNum); - - try { - producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), newAddLogicalQueue); - Assert.fail("write to decreased logical queue success, want it failed"); - } catch (MQBrokerException e) { - assertThat(e.getResponseCode()).isEqualTo(ResponseCode.NO_PERMISSION); - } - { - int offset = 0; - PullResult pullResult = consumer.pull(newAddLogicalQueue, "*", offset, 10); - assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND); - assertThat(pullResult.getMsgFoundList()).hasSize(MSG_SENT_TIMES); - for (int i = 0; i < MSG_SENT_TIMES; i++) { - MessageExt msg = pullResult.getMsgFoundList().get(i); - assertThat(msg.getBrokerName()).isEqualTo(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME); - assertThat(msg.getQueueId()).isEqualTo(newAddLogicalQueue.getQueueId()); - assertThat(new String(msg.getBody(), StandardCharsets.UTF_8)).isEqualTo(String.format(Locale.ENGLISH, "%s-%d-%d", methodName, newAddLogicalQueue.getQueueId(), i)); - assertThat(msg.getQueueOffset()).isEqualTo(offset + i); - } - } - - // rotate to remove new add queue's data, and try pull again - { - BrokerController brokerController = brokerControllerMap.get(newAddLogicalQueueBrokerName); - rotateBrokerCommitLog(brokerController); - deleteCommitLogFiles(brokerController, 1); - } - { - int offset = 0; - PullResult pullResult = consumer.pull(newAddLogicalQueue, "*", offset, 10); - assertThat(pullResult.getPullStatus()).isIn(PullStatus.NO_NEW_MSG, PullStatus.NO_MATCHED_MSG); - } - } - - @Test - public void test007_LogicalQueueWritableEvenBrokerDown() throws Exception { - final String methodName = getCurrentMethodName(); - - final int logicalQueueIdx = 1; - - BrokerController brokerController3 = IntegrationTestBase.createAndStartBroker(nsAddr); - String broker3Name = brokerController3.getBrokerConfig().getBrokerName(); - brokerControllerMap.put(broker3Name, brokerController3); - await().atMost(30, TimeUnit.SECONDS).until(() -> mqAdminExt.examineBrokerClusterInfo().getBrokerAddrTable().containsKey(broker3Name)); - mqAdminExt.createAndUpdateTopicConfig(brokerController3.getBrokerAddr(), new TopicConfig(topic, 0, 0, PermName.PERM_READ | PermName.PERM_WRITE)); - - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, brokerController3.getBrokerConfig().getBrokerName(), null); - - MessageQueue migrateMessageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx); - { - for (int i = 0; i < MSG_SENT_TIMES; i++) { - SendResult sendResult = producer.send(new Message(topic, String.format(Locale.ENGLISH, "%s-%d-%d", methodName, migrateMessageQueue.getQueueId(), i).getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migrateMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migrateMessageQueue.getQueueId()); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(broker3Name); - assertThat(sendResult2.getOrigQueueId()).isEqualTo(0); - } - } - brokerController3.shutdown(); - brokerControllerMap.remove(broker3Name); - - assertThatThrownBy(() -> { - SendResult sendResult = producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); - logger.error("send should fail but got {}", sendResult); - }).isInstanceOf(RemotingException.class).hasMessageMatching("connect to [0-9.:]+ failed"); - - assertThatThrownBy(() -> { - new MigrateTopicLogicalQueueCommand().execute(mqAdminExt, topic, logicalQueueIdx, broker1Name, null); - }).hasRootCauseInstanceOf(RemotingConnectException.class).hasMessageContaining("migrateTopicLogicalQueuePrepare"); - - { - SendResult sendResult = producer.send(new Message(topic, "aaa".getBytes(StandardCharsets.UTF_8)), migrateMessageQueue); - assertThat(sendResult.getMessageQueue().getBrokerName()).isEqualTo(migrateMessageQueue.getBrokerName()); - assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(migrateMessageQueue.getQueueId()); - assertThat(sendResult.getQueueOffset()).isEqualTo(-1); - SendResultForLogicalQueue sendResult2 = (SendResultForLogicalQueue) sendResult; - assertThat(sendResult2.getOrigBrokerName()).isEqualTo(broker1Name); - assertThat(sendResult2.getOrigQueueId()).isIn( - /* CommitLog not rotated, will not reuse */QUEUE_NUMBERS, - /* CommitLog rotated in other test cases, will reuse */logicalQueueIdx - ); - } - } - - private static String getBrokerCommitLogFileName(BrokerController brokerController) throws IllegalAccessException { - DefaultMessageStore defaultMessageStore = (DefaultMessageStore) brokerController.getMessageStore(); - MappedFileQueue mfq = (MappedFileQueue) FieldUtils.readDeclaredField(defaultMessageStore.getCommitLog(), "mappedFileQueue", true); - return mfq.getLastMappedFile().getFileName(); - } - - private static void deleteCommitLogFiles(BrokerController brokerController, - int keepNum) throws IllegalAccessException { - CommitLog commitLog = ((DefaultMessageStore) brokerController.getMessageStore()).getCommitLog(); - commitLog.flush(); - MappedFileQueue mfq = (MappedFileQueue) FieldUtils.readDeclaredField(commitLog, "mappedFileQueue", true); - AtomicInteger count = new AtomicInteger(); - waitAtMost(5, TimeUnit.SECONDS).until(() -> { - count.getAndAdd(commitLog.deleteExpiredFile(0, 0, 5000, true, 1)); - return mfq.getMappedFiles().size() <= keepNum; - }); - brokerController.getTopicConfigManager().getLogicalQueueCleanHook().execute((DefaultMessageStore) brokerController.getMessageStore(), count.get()); - logger.info("deleteCommitLogFiles {} count {}", brokerController.getBrokerConfig().getBrokerName(), count.get()); - } - - private static void rotateBrokerCommitLog(BrokerController brokerController) throws IllegalAccessException { - CommitLog commitLog = ((DefaultMessageStore) brokerController.getMessageStore()).getCommitLog(); - commitLog.flush(); - String brokerName = brokerController.getBrokerConfig().getBrokerName(); - String fileName1 = getBrokerCommitLogFileName(brokerController); - logger.info("rotateBrokerCommitLog {} first {}", brokerName, fileName1); - int msgSize = 4 * 1024; - byte[] data = RandomStringUtils.randomAscii(msgSize).getBytes(StandardCharsets.UTF_8); - Message msg = new Message(placeholderTopic, data); - MessageQueue mq = new MessageQueue(placeholderTopic, brokerName, 0); - waitAtMost(5, TimeUnit.SECONDS).until(() -> { - for (int i = 0; i < 128; i++) { - producer.send(msg, mq); - } - commitLog.flush(); - String fileName2 = getBrokerCommitLogFileName(brokerController); - if (!fileName1.equals(fileName2)) { - logger.info("rotateBrokerCommitLog {} 4K msg last {}", brokerName, fileName2); - return true; - } - return false; - }); - } - - private long maxOffsetUncommitted(MessageQueue mq) throws IllegalAccessException, MQClientException { - DefaultMQAdminExtImpl defaultMQAdminExtImpl = (DefaultMQAdminExtImpl) FieldUtils.readDeclaredField(mqAdminExt, "defaultMQAdminExtImpl", true); - return defaultMQAdminExtImpl.maxOffset(mq, false); - } -} diff --git a/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java new file mode 100644 index 0000000000..22443d738e --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java @@ -0,0 +1,465 @@ +package org.apache.rocketmq.test.statictopic; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.apache.log4j.Logger; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.admin.ConsumeStats; +import org.apache.rocketmq.common.admin.OffsetWrapper; +import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; +import org.apache.rocketmq.test.util.MQAdminTestUtils; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.apache.rocketmq.test.util.VerifyUtils; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminUtils; +import org.apache.rocketmq.tools.command.MQAdminStartup; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.google.common.truth.Truth.assertThat; +import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig; + +@FixMethodOrder +public class StaticTopicIT extends BaseConf { + + private static Logger logger = Logger.getLogger(StaticTopicIT.class); + private DefaultMQAdminExt defaultMQAdminExt; + + @Before + public void setUp() throws Exception { + System.setProperty("rocketmq.client.rebalance.waitInterval", "500"); + defaultMQAdminExt = getAdmin(nsAddr); + waitBrokerRegistered(nsAddr, clusterName); + defaultMQAdminExt.start(); + } + + @Test + public void testCommandsWithCluster() throws Exception { + //This case is used to mock the env to test the command manually + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); + int queueNum = 10; + int msgEachQueue = 100; + + { + MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, null, clusterName, nsAddr); + sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0); + //consume and check + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1); + } + { + MQAdminTestUtils.remappingStaticTopicWithCommand(topic, null, clusterName, nsAddr); + Thread.sleep(500); + sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 100); + } + } + + @Test + public void testCommandsWithBrokers() throws Exception { + //This case is used to mock the env to test the command manually + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); + int queueNum = 10; + int msgEachQueue = 100; + { + Set brokers = ImmutableSet.of(broker1Name); + MQAdminTestUtils.createStaticTopicWithCommand(topic, queueNum, brokers, null, nsAddr); + sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, 0); + //consume and check + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1); + } + { + Set brokers = ImmutableSet.of(broker2Name); + MQAdminTestUtils.remappingStaticTopicWithCommand(topic, brokers, null, nsAddr); + Thread.sleep(500); + sendMessagesAndCheck(producer, brokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE); + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2); + } + } + + @Test + public void testNoTargetBrokers() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + int queueNum = 10; + { + Set targetBrokers = new HashSet<>(); + targetBrokers.add(broker1Name); + MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt); + Map remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size()); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true); + Assert.assertEquals(queueNum, globalIdMap.size()); + TopicConfigAndQueueMapping configMapping = remoteBrokerConfigMap.get(broker2Name); + Assert.assertEquals(0, configMapping.getWriteQueueNums()); + Assert.assertEquals(0, configMapping.getReadQueueNums()); + Assert.assertEquals(0, configMapping.getMappingDetail().getHostedQueues().size()); + } + + { + Set targetBrokers = new HashSet<>(); + targetBrokers.add(broker2Name); + MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt); + Map remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size()); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true); + Assert.assertEquals(queueNum, globalIdMap.size()); + } + + } + + private void sendMessagesAndCheck(RMQNormalProducer producer, Set targetBrokers, String topic, int queueNum, int msgEachQueue, long baseOffset) throws Exception { + ClientMetadata clientMetadata = MQAdminUtils.getBrokerAndTopicMetadata(topic, defaultMQAdminExt); + List messageQueueList = producer.getMessageQueue(); + Assert.assertEquals(queueNum, messageQueueList.size()); + for (int i = 0; i < queueNum; i++) { + MessageQueue messageQueue = messageQueueList.get(i); + Assert.assertEquals(topic, messageQueue.getTopic()); + Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageQueue.getBrokerName()); + Assert.assertEquals(i, messageQueue.getQueueId()); + String destBrokerName = clientMetadata.getBrokerNameFromMessageQueue(messageQueue); + Assert.assertTrue(targetBrokers.contains(destBrokerName)); + } + for(MessageQueue messageQueue: messageQueueList) { + producer.send(msgEachQueue, messageQueue); + } + Assert.assertEquals(0, producer.getSendErrorMsg().size()); + //leave the time to build the cq + Thread.sleep(100); + for(MessageQueue messageQueue: messageQueueList) { + Assert.assertEquals(0, defaultMQAdminExt.minOffset(messageQueue)); + Assert.assertEquals(msgEachQueue + baseOffset, defaultMQAdminExt.maxOffset(messageQueue)); + } + TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + for(MessageQueue messageQueue: messageQueueList) { + Assert.assertEquals(0, topicStatsTable.getOffsetTable().get(messageQueue).getMinOffset()); + Assert.assertEquals(msgEachQueue + baseOffset, topicStatsTable.getOffsetTable().get(messageQueue).getMaxOffset()); + } + } + + private Map> computeMessageByQueue(Collection msgs) { + Map> messagesByQueue = new HashMap<>(); + for (Object object : msgs) { + MessageExt messageExt = (MessageExt) object; + if (!messagesByQueue.containsKey(messageExt.getQueueId())) { + messagesByQueue.put(messageExt.getQueueId(), new ArrayList<>()); + } + messagesByQueue.get(messageExt.getQueueId()).add(messageExt); + } + for (List msgEachQueue: messagesByQueue.values()) { + Collections.sort(msgEachQueue, new Comparator() { + @Override + public int compare(MessageExt o1, MessageExt o2) { + return (int) (o1.getQueueOffset() - o2.getQueueOffset()); + } + }); + } + return messagesByQueue; + } + + private void consumeMessagesAndCheck(RMQNormalProducer producer, RMQNormalConsumer consumer, String topic, int queueNum, int msgEachQueue, int startGen, int genNum) { + consumer.getListener().waitForMessageConsume(producer.getAllMsgBody(), 30000); + /*System.out.println("produce:" + producer.getAllMsgBody().size()); + System.out.println("consume:" + consumer.getListener().getAllMsgBody().size());*/ + + assertThat(VerifyUtils.getFilterdMessage(producer.getAllMsgBody(), + consumer.getListener().getAllMsgBody())) + .containsExactlyElementsIn(producer.getAllMsgBody()); + Map> messagesByQueue = computeMessageByQueue(consumer.getListener().getAllOriginMsg()); + Assert.assertEquals(queueNum, messagesByQueue.size()); + for (int i = 0; i < queueNum; i++) { + List messageExts = messagesByQueue.get(i); + /*for (MessageExt messageExt:messageExts) { + System.out.printf("%d %d\n", messageExt.getQueueId(), messageExt.getQueueOffset()); + }*/ + int totalEachQueue = msgEachQueue * genNum; + Assert.assertEquals(totalEachQueue, messageExts.size()); + for (int j = 0; j < totalEachQueue; j++) { + MessageExt messageExt = messageExts.get(j); + int currGen = startGen + j / msgEachQueue; + Assert.assertEquals(topic, messageExt.getTopic()); + Assert.assertEquals(TopicQueueMappingUtils.getMockBrokerName(MixAll.METADATA_SCOPE_GLOBAL), messageExt.getBrokerName()); + Assert.assertEquals(i, messageExt.getQueueId()); + Assert.assertEquals((j % msgEachQueue) + currGen * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, messageExt.getQueueOffset()); + } + } + } + + + @Test + public void testCreateProduceConsumeStaticTopic() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); + + int queueNum = 10; + int msgEachQueue = 100; + //create static topic + Map localBrokerConfigMap = MQAdminTestUtils.createStaticTopic(topic, queueNum, getBrokers(), defaultMQAdminExt); + //check the static topic config + { + Map remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + Assert.assertEquals(brokerNum, remoteBrokerConfigMap.size()); + for (Map.Entry entry: remoteBrokerConfigMap.entrySet()) { + String broker = entry.getKey(); + TopicConfigAndQueueMapping configMapping = entry.getValue(); + TopicConfigAndQueueMapping localConfigMapping = localBrokerConfigMap.get(broker); + Assert.assertNotNull(localConfigMapping); + Assert.assertEquals(configMapping, localConfigMapping); + } + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true); + Assert.assertEquals(queueNum, globalIdMap.size()); + } + //send and check + sendMessagesAndCheck(producer, getBrokers(), topic, queueNum, msgEachQueue, 0); + //consume and check + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1); + } + + + @Test + public void testRemappingProduceConsumeStaticTopic() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + RMQNormalConsumer consumer = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); + + int queueNum = 1; + int msgEachQueue = 100; + //create send consume + { + Set targetBrokers = ImmutableSet.of(broker1Name); + MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0); + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1); + } + //remapping the static topic + { + Set targetBrokers = ImmutableSet.of(broker2Name); + MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt); + Map remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true); + Assert.assertEquals(queueNum, globalIdMap.size()); + for (TopicQueueMappingOne mappingOne: globalIdMap.values()) { + Assert.assertEquals(broker2Name, mappingOne.getBname()); + Assert.assertEquals(TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset()); + } + Thread.sleep(500); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE); + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 2); + } + } + + + @Test + public void testDoubleReadCheckConsumerOffset() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + String group = initConsumerGroup(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener()); + + int queueNum = 10; + int msgEachQueue = 100; + //create static topic + { + Set targetBrokers = ImmutableSet.of(broker1Name); + MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0); + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 0, 1); + } + producer.shutdown(); + consumer.shutdown(); + //use a new producer + producer = getProducer(nsAddr, topic); + + ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(group); + List messageQueues = producer.getMessageQueue(); + for (MessageQueue queue: messageQueues) { + OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue); + Assert.assertNotNull(wrapper); + Assert.assertEquals(msgEachQueue, wrapper.getBrokerOffset()); + Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset()); + } + + List brokers = ImmutableList.of(broker2Name, broker3Name, broker1Name); + for (int i = 0; i < brokers.size(); i++) { + Set targetBrokers = ImmutableSet.of(brokers.get(i)); + MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt); + //make the metadata + Thread.sleep(500); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, (i + 1) * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE); + } + consumeStats = defaultMQAdminExt.examineConsumeStats(group); + + messageQueues = producer.getMessageQueue(); + for (MessageQueue queue: messageQueues) { + OffsetWrapper wrapper = consumeStats.getOffsetTable().get(queue); + Assert.assertNotNull(wrapper); + Assert.assertEquals(msgEachQueue + brokers.size() * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE, wrapper.getBrokerOffset()); + Assert.assertEquals(msgEachQueue, wrapper.getConsumerOffset()); + } + consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener()); + consumeMessagesAndCheck(producer, consumer, topic, queueNum, msgEachQueue, 1, brokers.size()); + } + + + @Test + public void testRemappingAndClear() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + int queueNum = 10; + int msgEachQueue = 100; + //create to broker1Name + { + Set targetBrokers = ImmutableSet.of(broker1Name); + MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt); + //leave the time to refresh the metadata + Thread.sleep(500); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0); + } + + //remapping to broker2Name + { + Set targetBrokers = ImmutableSet.of(broker2Name); + MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt); + //leave the time to refresh the metadata + Thread.sleep(500); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 1 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE); + } + + //remapping to broker3Name + { + Set targetBrokers = ImmutableSet.of(broker3Name); + MQAdminTestUtils.remappingStaticTopic(topic, targetBrokers, defaultMQAdminExt); + //leave the time to refresh the metadata + Thread.sleep(500); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 2 * TopicQueueMappingUtils.DEFAULT_BLOCK_SEQ_SIZE); + } + + // 1 -> 2 -> 3, currently 1 should not has any mappings + + { + for (int i = 0; i < 10; i++) { + for (BrokerController brokerController: brokerControllerList) { + brokerController.getTopicQueueMappingCleanService().wakeup(); + } + Thread.sleep(100); + } + Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + Assert.assertEquals(brokerNum, brokerConfigMap.size()); + TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name); + TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name); + TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name); + Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size()); + Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size()); + + Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size()); + + } + { + Set topics = new HashSet<>(brokerController1.getTopicConfigManager().getTopicConfigTable().keySet()); + topics.remove(topic); + brokerController1.getMessageStore().cleanUnusedTopic(topics); + brokerController2.getMessageStore().cleanUnusedTopic(topics); + for (int i = 0; i < 10; i++) { + for (BrokerController brokerController: brokerControllerList) { + brokerController.getTopicQueueMappingCleanService().wakeup(); + } + Thread.sleep(100); + } + + Map brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + Assert.assertEquals(brokerNum, brokerConfigMap.size()); + TopicConfigAndQueueMapping config1 = brokerConfigMap.get(broker1Name); + TopicConfigAndQueueMapping config2 = brokerConfigMap.get(broker2Name); + TopicConfigAndQueueMapping config3 = brokerConfigMap.get(broker3Name); + Assert.assertEquals(0, config1.getMappingDetail().getHostedQueues().size()); + Assert.assertEquals(queueNum, config2.getMappingDetail().getHostedQueues().size()); + Assert.assertEquals(queueNum, config3.getMappingDetail().getHostedQueues().size()); + //The first leader will clear it + for (List items : config1.getMappingDetail().getHostedQueues().values()) { + Assert.assertEquals(3, items.size()); + } + //The second leader do nothing + for (List items : config3.getMappingDetail().getHostedQueues().values()) { + Assert.assertEquals(1, items.size()); + } + } + } + + + @Test + public void testRemappingWithNegativeLogicOffset() throws Exception { + String topic = "static" + MQRandomUtils.getRandomTopic(); + RMQNormalProducer producer = getProducer(nsAddr, topic); + int queueNum = 10; + int msgEachQueue = 100; + //create and send + { + Set targetBrokers = ImmutableSet.of(broker1Name); + MQAdminTestUtils.createStaticTopic(topic, queueNum, targetBrokers, defaultMQAdminExt); + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0); + } + + //remapping the static topic with -1 logic offset + { + Set targetBrokers = ImmutableSet.of(broker2Name); + MQAdminTestUtils.remappingStaticTopicWithNegativeLogicOffset(topic, targetBrokers, defaultMQAdminExt); + Map remoteBrokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, remoteBrokerConfigMap); + Map globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(getMappingDetailFromConfig(remoteBrokerConfigMap.values())), false, true); + Assert.assertEquals(queueNum, globalIdMap.size()); + for (TopicQueueMappingOne mappingOne: globalIdMap.values()) { + Assert.assertEquals(broker2Name, mappingOne.getBname()); + Assert.assertEquals(-1, mappingOne.getItems().get(mappingOne.getItems().size() - 1).getLogicOffset()); + } + //leave the time to refresh the metadata + Thread.sleep(500); + //here the gen should be 0 + sendMessagesAndCheck(producer, targetBrokers, topic, queueNum, msgEachQueue, 0); + } + } + + + @After + public void tearDown() { + System.setProperty("rocketmq.client.rebalance.waitInterval", "20000"); + super.shutdown(); + } + +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index d17c717ea1..6f551d36b1 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -16,11 +16,6 @@ */ package org.apache.rocketmq.tools.admin; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; @@ -43,17 +38,14 @@ import org.apache.rocketmq.common.protocol.body.ConsumerConnection; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.remoting.RPCHook; @@ -64,6 +56,12 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { private final DefaultMQAdminExtImpl defaultMQAdminExtImpl; private String adminExtGroup = "admin_ext_group"; @@ -215,7 +213,7 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { @Override public TopicConfig examineTopicConfig(String addr, - String topic) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException { + String topic) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, InterruptedException, MQBrokerException { return defaultMQAdminExtImpl.examineTopicConfig(addr, topic); } @@ -226,6 +224,12 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { return defaultMQAdminExtImpl.examineTopicStats(topic); } + @Override + public TopicStatsTable examineTopicStats(String brokerAddr, String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return defaultMQAdminExtImpl.examineTopicStats(brokerAddr, topic); + } + @Override public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { return this.defaultMQAdminExtImpl.fetchAllTopicList(); @@ -606,56 +610,9 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { } @Override - public void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, - int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - this.defaultMQAdminExtImpl.updateTopicLogicalQueueMapping(brokerAddr, topic, queueId, logicalQueueIndex); + public void createStaticTopic(String addr, String defaultTopic, TopicConfig topicConfig, TopicQueueMappingDetail mappingDetail, boolean force) throws RemotingException, InterruptedException, MQBrokerException { + this.defaultMQAdminExtImpl.createStaticTopic(addr, defaultTopic, topicConfig, mappingDetail, force); } - @Override - public LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, - String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { - return this.defaultMQAdminExtImpl.queryTopicLogicalQueueMapping(brokerAddr, topic); - } - @Override - public void deleteTopicLogicalQueueMapping(String brokerAddr, - String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - this.defaultMQAdminExtImpl.deleteTopicLogicalQueueMapping(brokerAddr, topic); - } - - @Override - public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, - LogicalQueueRouteData queueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { - return this.defaultMQAdminExtImpl.sealTopicLogicalQueue(brokerAddr, queueRouteData); - } - - @Override public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, - int logicalQueueIdx, - MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.defaultMQAdminExtImpl.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, messageQueueRouteState); - } - - @Override public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, - int logicalQueueIdx, - MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.defaultMQAdminExtImpl.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, messageQueueStatus); - } - - @Override public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare( - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.defaultMQAdminExtImpl.migrateTopicLogicalQueuePrepare(fromQueueRouteData, toQueueRouteData); - } - - @Override public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.defaultMQAdminExtImpl.migrateTopicLogicalQueueCommit(fromQueueRouteData, toQueueRouteData); - } - - @Override public void migrateTopicLogicalQueueNotify(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - this.defaultMQAdminExtImpl.migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData); - } } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 2f2e49130a..b7e4816de7 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -66,7 +66,6 @@ import org.apache.rocketmq.common.protocol.body.ConsumerConnection; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; @@ -76,11 +75,13 @@ import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; @@ -94,15 +95,10 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; import org.apache.rocketmq.tools.admin.api.TrackType; -public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { +import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.checkAndBuildMappingItems; +import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig; - private final InternalLogger log = ClientLogger.getLog(); - private final DefaultMQAdminExt defaultMQAdminExt; - private ServiceState serviceState = ServiceState.CREATE_JUST; - private MQClientInstance mqClientInstance; - private RPCHook rpcHook; - private long timeoutMillis = 20000; - private Random random = new Random(); +public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { private static final Set SYSTEM_GROUP_SET = new HashSet(); @@ -122,6 +118,14 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { SYSTEM_GROUP_SET.add(MixAll.CID_SYS_RMQ_TRANS); } + private final InternalLogger log = ClientLogger.getLog(); + private final DefaultMQAdminExt defaultMQAdminExt; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mqClientInstance; + private RPCHook rpcHook; + private long timeoutMillis = 20000; + private Random random = new Random(); + public DefaultMQAdminExtImpl(DefaultMQAdminExt defaultMQAdminExt, long timeoutMillis) { this(defaultMQAdminExt, null, timeoutMillis); } @@ -247,7 +251,8 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } @Override - public TopicConfig examineTopicConfig(String addr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + public TopicConfig examineTopicConfig(String addr, + String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { return this.mqClientInstance.getMQClientAPIImpl().getTopicConfig(addr, topic, timeoutMillis); } @@ -266,6 +271,10 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } } + //Get the static stats + Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(topic, topicRouteData, defaultMQAdminExt); + MQAdminUtils.convertPhysicalTopicStats(topic, brokerConfigMap, topicStatsTable); + if (topicStatsTable.getOffsetTable().isEmpty()) { throw new MQClientException("Not found the topic stats info", null); } @@ -273,6 +282,13 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { return topicStatsTable; } + @Override + public TopicStatsTable examineTopicStats(String brokerAddr, + String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicStatsInfo(brokerAddr, topic, timeoutMillis); + } + @Override public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { return this.mqClientInstance.getMQClientAPIImpl().getTopicListFromNameServer(timeoutMillis); @@ -317,12 +333,29 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } } - if (result.getOffsetTable().isEmpty()) { + Set topics = new HashSet<>(); + for (MessageQueue messageQueue: result.getOffsetTable().keySet()) { + topics.add(messageQueue.getTopic()); + } + + ConsumeStats staticResult = new ConsumeStats(); + staticResult.setConsumeTps(result.getConsumeTps()); + // for topic, we put the physical stats, how about group? + // staticResult.getOffsetTable().putAll(result.getOffsetTable()); + + for (String currentTopic: topics) { + TopicRouteData currentRoute = this.examineTopicRouteInfo(currentTopic); + Map brokerConfigMap = MQAdminUtils.examineTopicConfigFromRoute(currentTopic, currentRoute, defaultMQAdminExt); + ConsumeStats consumeStats = MQAdminUtils.convertPhysicalConsumeStats(brokerConfigMap, result); + staticResult.getOffsetTable().putAll(consumeStats.getOffsetTable()); + } + + if (staticResult.getOffsetTable().isEmpty()) { throw new MQClientException(ResponseCode.CONSUMER_NOT_ONLINE, "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message"); } - return result; + return staticResult; } @Override @@ -349,54 +382,6 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, msgId); } - @Override - public void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { - this.mqClientInstance.getMQClientAPIImpl().updateTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIndex, timeoutMillis); - } - - @Override - public LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().queryTopicLogicalQueue(brokerAddr, topic, timeoutMillis); - } - - @Override - public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { - this.mqClientInstance.getMQClientAPIImpl().deleteTopicLogicalQueueMapping(brokerAddr, topic, timeoutMillis); - } - - @Override - public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().sealTopicLogicalQueue(brokerAddr, queueRouteData, timeoutMillis); - } - - @Override public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, - int logicalQueueIdx, MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, messageQueueRouteState, timeoutMillis); - } - - @Override public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, - int logicalQueueIdx, MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, messageQueueStatus, timeoutMillis); - } - - @Override public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare( - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueuePrepare(fromQueueRouteData.getBrokerAddr(), fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - - @Override public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - return this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueueCommit(toQueueRouteData.getBrokerAddr(), fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - - @Override public void migrateTopicLogicalQueueNotify(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { - this.mqClientInstance.getMQClientAPIImpl().migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis); - } - @Override public ConsumerConnection examineConsumerConnectionInfo( String consumerGroup) throws InterruptedException, MQBrokerException, @@ -457,7 +442,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { @Override public int addWritePermOfBroker(String namesrvAddr, String brokerName) throws RemotingCommandException, - RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException { + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQClientException { return this.mqClientInstance.getMQClientAPIImpl().addWritePermOfBroker(namesrvAddr, brokerName, timeoutMillis); } @@ -1087,6 +1072,13 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { this.mqClientInstance.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); } + @Override + public void createStaticTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, + final TopicQueueMappingDetail mappingDetail, + final boolean force) throws RemotingException, InterruptedException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().createStaticTopic(addr, defaultTopic, topicConfig, mappingDetail, force, timeoutMillis); + } + @Override public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { return this.mqClientInstance.getMQAdminImpl().searchOffset(mq, timestamp); @@ -1097,10 +1089,6 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { return this.mqClientInstance.getMQAdminImpl().maxOffset(mq); } - public long maxOffset(MessageQueue mq, boolean committed) throws MQClientException { - return this.mqClientInstance.getMQAdminImpl().maxOffset(mq, committed); - } - @Override public long minOffset(MessageQueue mq) throws MQClientException { return this.mqClientInstance.getMQAdminImpl().minOffset(mq); @@ -1188,7 +1176,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { @Override public void setMessageRequestMode(final String brokerAddr, final String topic, final String consumerGroup, final - MessageRequestMode mode, final int popShareQueueNum, final long timeoutMillis) + MessageRequestMode mode, final int popShareQueueNum, final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException { this.mqClientInstance.getMQClientAPIImpl().setMessageRequestMode(brokerAddr, topic, consumerGroup, mode, popShareQueueNum, timeoutMillis); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java index 5c53ead609..35efe9629f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminExt.java @@ -16,11 +16,6 @@ */ package org.apache.rocketmq.tools.admin; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; import org.apache.rocketmq.client.MQAdmin; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -42,7 +37,6 @@ import org.apache.rocketmq.common.protocol.body.ConsumerConnection; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.GroupList; import org.apache.rocketmq.common.protocol.body.KVTable; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; import org.apache.rocketmq.common.protocol.body.ProducerConnection; import org.apache.rocketmq.common.protocol.body.QueryConsumeQueueResponseBody; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; @@ -50,9 +44,7 @@ import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.protocol.body.TopicList; import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingConnectException; @@ -61,6 +53,12 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + public interface MQAdminExt extends MQAdmin { void start() throws MQClientException; @@ -100,13 +98,14 @@ public interface MQAdminExt extends MQAdmin { SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr, final String group) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; - TopicConfig examineTopicConfig(final String addr, - final String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; TopicStatsTable examineTopicStats( final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + TopicStatsTable examineTopicStats(String brokerAddr, final String topic) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException; + TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException; TopicList fetchTopicsByCLuster( @@ -312,27 +311,11 @@ public interface MQAdminExt extends MQAdmin { throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQClientException; - void updateTopicLogicalQueueMapping(String brokerAddr, String topic, int queueId, int logicalQueueIndex) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException; - LogicalQueuesInfo queryTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - void deleteTopicLogicalQueueMapping(String brokerAddr, String topic) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException; + TopicConfig examineTopicConfig(final String addr, + final String topic) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + void createStaticTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final TopicQueueMappingDetail mappingDetail, final boolean force) throws RemotingException, InterruptedException, MQBrokerException; - LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId, int logicalQueueIdx, - MessageQueueRouteState messageQueueRouteState) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - - LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, int logicalQueueIdx, - MessageQueueRouteState messageQueueStatus) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - - MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare(LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - - MigrateLogicalQueueBody migrateTopicLogicalQueueCommit( - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; - - void migrateTopicLogicalQueueNotify(String brokerAddr, - LogicalQueueRouteData fromQueueRouteData, - LogicalQueueRouteData toQueueRouteData) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException; } diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminUtils.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminUtils.java new file mode 100644 index 0000000000..d915cb1e72 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/MQAdminUtils.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.admin; + +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.admin.ConsumeStats; +import org.apache.rocketmq.common.admin.OffsetWrapper; +import org.apache.rocketmq.common.admin.TopicOffset; +import org.apache.rocketmq.common.admin.TopicStatsTable; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.remoting.exception.RemotingConnectException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.checkAndBuildMappingItems; +import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig; + +public class MQAdminUtils { + + + public static ClientMetadata getBrokerMetadata(DefaultMQAdminExt defaultMQAdminExt) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + ClientMetadata clientMetadata = new ClientMetadata(); + refreshClusterInfo(defaultMQAdminExt, clientMetadata); + return clientMetadata; + } + + public static ClientMetadata getBrokerAndTopicMetadata(String topic, DefaultMQAdminExt defaultMQAdminExt) throws InterruptedException, RemotingException, MQBrokerException { + ClientMetadata clientMetadata = new ClientMetadata(); + refreshClusterInfo(defaultMQAdminExt, clientMetadata); + refreshTopicRouteInfo(topic, defaultMQAdminExt, clientMetadata); + return clientMetadata; + } + + public static void refreshClusterInfo(DefaultMQAdminExt defaultMQAdminExt, ClientMetadata clientMetadata) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + if (clusterInfo == null + || clusterInfo.getClusterAddrTable().isEmpty()) { + throw new RuntimeException("The Cluster info is empty"); + } + clientMetadata.refreshClusterInfo(clusterInfo); + } + + public static void refreshTopicRouteInfo(String topic, DefaultMQAdminExt defaultMQAdminExt, ClientMetadata clientMetadata) throws RemotingException, InterruptedException, MQBrokerException { + TopicRouteData routeData = null; + try { + routeData = defaultMQAdminExt.examineTopicRouteInfo(topic); + } catch (MQClientException exception) { + if (exception.getResponseCode() != ResponseCode.TOPIC_NOT_EXIST) { + throw new MQBrokerException(exception.getResponseCode(), exception.getErrorMessage()); + } + } + if (routeData != null + && !routeData.getQueueDatas().isEmpty()) { + clientMetadata.freshTopicRoute(topic, routeData); + } + } + + public static Set getAllBrokersInSameCluster(Collection brokers, DefaultMQAdminExt defaultMQAdminExt) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + if (clusterInfo == null + || clusterInfo.getClusterAddrTable().isEmpty()) { + throw new RuntimeException("The Cluster info is empty"); + } + Set allBrokers = new HashSet<>(); + for (String broker: brokers) { + if (allBrokers.contains(broker)) { + continue; + } + for (Set clusterBrokers : clusterInfo.getClusterAddrTable().values()) { + if (clusterBrokers.contains(broker)) { + allBrokers.addAll(clusterBrokers); + break; + } + } + } + return allBrokers; + } + + public static void completeNoTargetBrokers(Map brokerConfigMap, DefaultMQAdminExt defaultMQAdminExt) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + TopicConfigAndQueueMapping configMapping = brokerConfigMap.values().iterator().next(); + String topic = configMapping.getTopicName(); + int queueNum = configMapping.getMappingDetail().getTotalQueues(); + long newEpoch = configMapping.getMappingDetail().getEpoch(); + Set allBrokers = getAllBrokersInSameCluster(brokerConfigMap.keySet(), defaultMQAdminExt); + for (String broker: allBrokers) { + if (!brokerConfigMap.containsKey(broker)) { + brokerConfigMap.put(broker, new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), new TopicQueueMappingDetail(topic, queueNum, broker, newEpoch))); + } + } + } + + public static void checkIfMasterAlive(Collection brokers, DefaultMQAdminExt defaultMQAdminExt, ClientMetadata clientMetadata) { + for (String broker : brokers) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + if (addr == null) { + throw new RuntimeException("Can't find addr for broker " + broker); + } + } + } + + public static void updateTopicConfigMappingAll(Map brokerConfigMap, DefaultMQAdminExt defaultMQAdminExt, boolean force) throws Exception { + ClientMetadata clientMetadata = getBrokerMetadata(defaultMQAdminExt); + checkIfMasterAlive(brokerConfigMap.keySet(), defaultMQAdminExt, clientMetadata); + //If some succeed, and others fail, it will cause inconsistent data + for (Map.Entry entry : brokerConfigMap.entrySet()) { + String broker = entry.getKey(); + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = entry.getValue(); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + } + + public static void remappingStaticTopic(String topic, Set brokersToMapIn, Set brokersToMapOut, Map brokerConfigMap, int blockSeqSize, boolean force, DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + + ClientMetadata clientMetadata = MQAdminUtils.getBrokerMetadata(defaultMQAdminExt); + MQAdminUtils.checkIfMasterAlive(brokerConfigMap.keySet(), defaultMQAdminExt, clientMetadata); + // now do the remapping + //Step1: let the new leader can be write without the logicOffset + for (String broker: brokersToMapIn) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + //Step2: forbid the write of old leader + for (String broker: brokersToMapOut) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + //Step3: decide the logic offset + for (String broker: brokersToMapOut) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicStatsTable statsTable = defaultMQAdminExt.examineTopicStats(addr, topic); + TopicConfigAndQueueMapping mapOutConfig = brokerConfigMap.get(broker); + for (Map.Entry> entry : mapOutConfig.getMappingDetail().getHostedQueues().entrySet()) { + List items = entry.getValue(); + Integer globalId = entry.getKey(); + if (items.size() < 2) { + continue; + } + LogicQueueMappingItem newLeader = items.get(items.size() - 1); + LogicQueueMappingItem oldLeader = items.get(items.size() - 2); + if (newLeader.getLogicOffset() > 0) { + continue; + } + TopicOffset topicOffset = statsTable.getOffsetTable().get(new MessageQueue(topic, oldLeader.getBname(), oldLeader.getQueueId())); + if (topicOffset == null) { + throw new RuntimeException("Cannot get the max offset for old leader " + oldLeader); + } + //TO DO check the max offset, will it return -1? + if (topicOffset.getMaxOffset() < oldLeader.getStartOffset()) { + throw new RuntimeException("The max offset is smaller then the start offset " + oldLeader + " " + topicOffset.getMaxOffset()); + } + newLeader.setLogicOffset(TopicQueueMappingUtils.blockSeqRoundUp(oldLeader.computeStaticQueueOffsetStrictly(topicOffset.getMaxOffset()), blockSeqSize)); + TopicConfigAndQueueMapping mapInConfig = brokerConfigMap.get(newLeader.getBname()); + //fresh the new leader + TopicQueueMappingDetail.putMappingInfo(mapInConfig.getMappingDetail(), globalId, items); + } + } + //Step4: write to the new leader with logic offset + for (String broker: brokersToMapIn) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + //Step5: write the non-target brokers + for (String broker: brokerConfigMap.keySet()) { + if (brokersToMapIn.contains(broker) || brokersToMapOut.contains(broker)) { + continue; + } + String addr = clientMetadata.findMasterBrokerAddr(broker); + TopicConfigAndQueueMapping configMapping = brokerConfigMap.get(broker); + defaultMQAdminExt.createStaticTopic(addr, defaultMQAdminExt.getCreateTopicKey(), configMapping, configMapping.getMappingDetail(), force); + } + } + + public static Map examineTopicConfigAll(String topic, DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, InterruptedException, MQBrokerException { + Map brokerConfigMap = new HashMap<>(); + ClientMetadata clientMetadata = new ClientMetadata(); + //check all the brokers + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + if (clusterInfo != null + && clusterInfo.getBrokerAddrTable() != null) { + clientMetadata.refreshClusterInfo(clusterInfo); + } + for (String broker : clientMetadata.getBrokerAddrTable().keySet()) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + try { + TopicConfigAndQueueMapping mapping = (TopicConfigAndQueueMapping) defaultMQAdminExt.examineTopicConfig(addr, topic); + //allow the config is null + if (mapping != null) { + if (mapping.getMappingDetail() != null) { + assert mapping.getMappingDetail().getBname().equals(broker); + } + brokerConfigMap.put(broker, mapping); + } + } catch (MQBrokerException exception1) { + if (exception1.getResponseCode() != ResponseCode.TOPIC_NOT_EXIST) { + throw exception1; + } + } + } + return brokerConfigMap; + } + + + public static Map examineTopicConfigFromRoute(String topic, TopicRouteData topicRouteData, DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, InterruptedException, MQBrokerException { + Map brokerConfigMap = new HashMap<>(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String broker = bd.getBrokerName(); + String addr = bd.selectBrokerAddr(); + if (addr == null) { + continue; + } + try { + TopicConfigAndQueueMapping mapping = (TopicConfigAndQueueMapping) defaultMQAdminExt.examineTopicConfig(addr, topic); + //allow the config is null + if (mapping != null) { + if (mapping.getMappingDetail() != null) { + assert mapping.getMappingDetail().getBname().equals(broker); + } + brokerConfigMap.put(broker, mapping); + } + } catch (MQBrokerException exception) { + if (exception.getResponseCode() != ResponseCode.TOPIC_NOT_EXIST) { + throw exception; + } + } + } + return brokerConfigMap; + } + + public static void convertPhysicalTopicStats(String topic, Map brokerConfigMap, TopicStatsTable topicStatsTable) { + Map globalIdMap = checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), true, false); + for (Map.Entry entry: globalIdMap.entrySet()) { + Integer qid = entry.getKey(); + TopicQueueMappingOne mappingOne = entry.getValue(); + LogicQueueMappingItem minItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingOne.getItems(), 0, true); + LogicQueueMappingItem maxItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingOne.getItems(), Long.MAX_VALUE, true); + assert minItem != null && maxItem != null; + TopicOffset minTopicOffset = topicStatsTable.getOffsetTable().get(new MessageQueue(topic, minItem.getBname(), minItem.getQueueId())); + TopicOffset maxTopicOffset = topicStatsTable.getOffsetTable().get(new MessageQueue(topic, maxItem.getBname(), maxItem.getQueueId())); + + if (minTopicOffset == null + || maxTopicOffset == null) { + continue; + } + long min = minItem.computeStaticQueueOffsetLoosely(minTopicOffset.getMinOffset()); + if (min < 0) + min = 0; + long max = maxItem.computeStaticQueueOffsetStrictly(maxTopicOffset.getMaxOffset()); + if (max < 0) + max = 0; + long timestamp = maxTopicOffset.getLastUpdateTimestamp(); + + TopicOffset topicOffset = new TopicOffset(); + topicOffset.setMinOffset(min); + topicOffset.setMaxOffset(max); + topicOffset.setLastUpdateTimestamp(timestamp); + topicStatsTable.getOffsetTable().put(new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(mappingOne.getMappingDetail().getScope()), qid), topicOffset); + } + } + + + public static ConsumeStats convertPhysicalConsumeStats(Map brokerConfigMap, ConsumeStats physicalResult) { + Map globalIdMap = checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), true, false); + ConsumeStats result = new ConsumeStats(); + result.setConsumeTps(physicalResult.getConsumeTps()); + for (Map.Entry entry : globalIdMap.entrySet()) { + Integer qid = entry.getKey(); + TopicQueueMappingOne mappingOne = entry.getValue(); + MessageQueue messageQueue = new MessageQueue(mappingOne.getTopic(), TopicQueueMappingUtils.getMockBrokerName(mappingOne.getMappingDetail().getScope()), qid); + OffsetWrapper offsetWrapper = new OffsetWrapper(); + long brokerOffset = -1; + long consumerOffset = -1; + long lastTimestamp = -1; //maybe need to be polished + for (int i = mappingOne.getItems().size() - 1; i >= 0; i--) { + LogicQueueMappingItem item = mappingOne.getItems().get(i); + MessageQueue phyQueue = new MessageQueue(mappingOne.getTopic(), item.getBname(), item.getQueueId()); + OffsetWrapper phyOffsetWrapper = physicalResult.getOffsetTable().get(phyQueue); + if (phyOffsetWrapper == null) { + continue; + } + + if (consumerOffset == -1 + && phyOffsetWrapper.getConsumerOffset() >= 0) { + consumerOffset = phyOffsetWrapper.getConsumerOffset(); + lastTimestamp = phyOffsetWrapper.getLastTimestamp(); + } + if (brokerOffset == -1 + && item.getLogicOffset() >= 0) { + brokerOffset = item.computeStaticQueueOffsetStrictly(phyOffsetWrapper.getBrokerOffset()); + } + if (consumerOffset >= 0 + && brokerOffset >= 0) { + break; + } + } + if (brokerOffset >= 0 + && consumerOffset >= 0) { + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + offsetWrapper.setLastTimestamp(lastTimestamp); + result.getOffsetTable().put(messageQueue, offsetWrapper); + } + } + return result; + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java index 7bc1aa81ce..bfac860d08 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java @@ -19,8 +19,6 @@ package org.apache.rocketmq.tools.command; import ch.qos.logback.classic.LoggerContext; import ch.qos.logback.classic.joran.JoranConfigurator; import ch.qos.logback.core.joran.spi.JoranException; -import java.util.ArrayList; -import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; @@ -52,13 +50,9 @@ import org.apache.rocketmq.tools.command.consumer.DeleteSubscriptionGroupCommand import org.apache.rocketmq.tools.command.consumer.GetConsumerConfigSubCommand; import org.apache.rocketmq.tools.command.consumer.StartMonitoringSubCommand; import org.apache.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand; -import org.apache.rocketmq.tools.command.logicalqueue.DeleteTopicLogicalQueueMappingCommand; -import org.apache.rocketmq.tools.command.logicalqueue.MigrateTopicLogicalQueueCommand; -import org.apache.rocketmq.tools.command.logicalqueue.QueryTopicLogicalQueueMappingCommand; -import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; -import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueNumCommand; -import org.apache.rocketmq.tools.command.export.ExportMetricsCommand; import org.apache.rocketmq.tools.command.export.ExportConfigsCommand; +import org.apache.rocketmq.tools.command.export.ExportMetadataCommand; +import org.apache.rocketmq.tools.command.export.ExportMetricsCommand; import org.apache.rocketmq.tools.command.message.CheckMsgSendRTCommand; import org.apache.rocketmq.tools.command.message.ConsumeMessageCommand; import org.apache.rocketmq.tools.command.message.PrintMessageByQueueCommand; @@ -81,16 +75,20 @@ import org.apache.rocketmq.tools.command.queue.QueryConsumeQueueCommand; import org.apache.rocketmq.tools.command.stats.StatsAllSubCommand; import org.apache.rocketmq.tools.command.topic.AllocateMQSubCommand; import org.apache.rocketmq.tools.command.topic.DeleteTopicSubCommand; -import org.apache.rocketmq.tools.command.export.ExportMetadataCommand; +import org.apache.rocketmq.tools.command.topic.RemappingStaticTopicSubCommand; import org.apache.rocketmq.tools.command.topic.TopicClusterSubCommand; import org.apache.rocketmq.tools.command.topic.TopicListSubCommand; import org.apache.rocketmq.tools.command.topic.TopicRouteSubCommand; import org.apache.rocketmq.tools.command.topic.TopicStatusSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateOrderConfCommand; +import org.apache.rocketmq.tools.command.topic.UpdateStaticTopicSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateTopicPermSubCommand; import org.apache.rocketmq.tools.command.topic.UpdateTopicSubCommand; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + public class MQAdminStartup { protected static List subCommandList = new ArrayList(); @@ -226,11 +224,9 @@ public class MQAdminStartup { initCommand(new UpdateGlobalWhiteAddrSubCommand()); initCommand(new GetAccessConfigSubCommand()); - initCommand(new UpdateTopicLogicalQueueMappingCommand()); - initCommand(new DeleteTopicLogicalQueueMappingCommand()); - initCommand(new QueryTopicLogicalQueueMappingCommand()); - initCommand(new MigrateTopicLogicalQueueCommand()); - initCommand(new UpdateTopicLogicalQueueNumCommand()); + + initCommand(new UpdateStaticTopicSubCommand()); + initCommand(new RemappingStaticTopicSubCommand()); initCommand(new ExportMetadataCommand()); initCommand(new ExportConfigsCommand()); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java deleted file mode 100644 index 77eb918b0e..0000000000 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/DeleteTopicLogicalQueueMappingCommand.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.tools.command.logicalqueue; - -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.srvutil.ServerUtil; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; -import org.apache.rocketmq.tools.command.SubCommand; -import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DeleteTopicLogicalQueueMappingCommand implements SubCommand { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); - - @Override public String commandName() { - return "deleteTopicLogicalQueueMapping"; - } - - @Override public String commandDesc() { - return "delete logical queue mapping info of a topic"; - } - - @Override public Options buildCommandlineOptions(Options options) { - Option opt; - - opt = new Option("t", "topic", true, "topic name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("c", "clusterName", true, "cluster name."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("b", "brokerAddr", true, "broker addr."); - opt.setRequired(false); - options.addOption(opt); - - return options; - } - - @Override public void execute(CommandLine commandLine, Options options, - RPCHook rpcHook) throws SubCommandException { - DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); - defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); - - try { - String topic = commandLine.getOptionValue("t").trim(); - List brokerAddrs; - if (commandLine.hasOption("b")) { - brokerAddrs = Collections.singletonList(commandLine.getOptionValue("c").trim()); - } else if (commandLine.hasOption("c")) { - String clusterName = commandLine.getOptionValue("c").trim(); - brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); - } else { - ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); - return; - } - for (String brokerAddr : brokerAddrs) { - log.info("deleteTopicLogicalQueueMapping {} {}", brokerAddr, topic); - defaultMQAdminExt.deleteTopicLogicalQueueMapping(brokerAddr, topic); - } - } catch (Exception e) { - throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); - } finally { - defaultMQAdminExt.shutdown(); - } - } -} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java deleted file mode 100644 index 5da8b0d480..0000000000 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/MigrateTopicLogicalQueueCommand.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.tools.command.logicalqueue; - -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.SubCommand; -import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static java.util.Optional.ofNullable; - -public class MigrateTopicLogicalQueueCommand implements SubCommand { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); - - @Override public String commandName() { - return "migrateTopicLogicalQueue"; - } - - @Override public String commandDesc() { - return "migrate a logical queue of a topic from one broker to another."; - } - - @Override public Options buildCommandlineOptions(Options options) { - Option opt; - - opt = new Option("t", "topic", true, "topic name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("i", "index", true, "logical queue index."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("b", "brokerAddr", true, "new broker name"); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("fd", "forceDelta", true, "assume fromBroker down, force migrate"); - opt.setRequired(false); - options.addOption(opt); - - return options; - } - - public void execute(DefaultMQAdminExt mqAdminExt, String topic, int logicalQueueIdx, - String toBrokerName, - Long forceDelta) throws RemotingException, MQBrokerException, InterruptedException, SubCommandException, MQClientException { - TopicRouteData topicRouteInfo = mqAdminExt.examineTopicRouteInfo(topic); - LogicalQueuesInfo logicalQueuesInfo = topicRouteInfo.getLogicalQueuesInfo(); - if (logicalQueuesInfo == null) { - throw new SubCommandException("topic not enabled logical queue"); - } - List queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx); - if (queueRouteDataList == null) { - throw new SubCommandException("logical queue %d not exist", logicalQueueIdx); - } - Map brokerAddrTable = mqAdminExt.examineBrokerClusterInfo().getBrokerAddrTable(); - String toBrokerAddr = lookupBrokerMasterAddr(brokerAddrTable, toBrokerName); - if (toBrokerAddr == null) { - throw new SubCommandException("destination broker %s not found", toBrokerName); - } - LogicalQueueRouteData fromQueueRouteData = queueRouteDataList.stream().filter(LogicalQueueRouteData::isWritable).reduce((first, second) -> second).orElse(null); - if (fromQueueRouteData == null) { - throw new SubCommandException("logical queue %d not writable, can not migrate", logicalQueueIdx); - } - String fromBrokerName = fromQueueRouteData.getBrokerName(); - String fromBrokerAddr = ofNullable(lookupBrokerMasterAddr(brokerAddrTable, fromBrokerName)).orElse(fromQueueRouteData.getBrokerAddr()); - if (fromBrokerAddr == null) { - throw new SubCommandException("unexpected source broker name %s not found", fromBrokerName); - } - LogicalQueueRouteData toQueueRouteData; - RETRY: - while (true) { - TopicConfig topicConfig = mqAdminExt.examineTopicConfig(toBrokerAddr, topic); - LogicalQueuesInfo logicalQueuesInfoInBroker = ofNullable(mqAdminExt.queryTopicLogicalQueueMapping(toBrokerAddr, topic)).orElse(new LogicalQueuesInfo()); - toQueueRouteData = logicalQueuesInfoInBroker.getOrDefault(logicalQueueIdx, Collections.emptyList()).stream().filter(queueRouteData -> Objects.equals(toBrokerName, queueRouteData.getBrokerName()) && queueRouteData.isWriteOnly()).findFirst().orElse(null); - if (toQueueRouteData == null) { - Multimap m = Multimaps.index(logicalQueuesInfoInBroker.values().stream().flatMap(Collection::stream).filter(queueRouteData -> Objects.equals(toBrokerName, queueRouteData.getBrokerName())).iterator(), LogicalQueueRouteData::getQueueId); - for (int queueId = 0, writeQueueNums = topicConfig.getWriteQueueNums(); queueId < writeQueueNums; queueId++) { - if (m.get(queueId).stream().anyMatch(LogicalQueueRouteData::isWritable)) { - continue; - } - try { - toQueueRouteData = mqAdminExt.reuseTopicLogicalQueue(toBrokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.WriteOnly); - log.info("reuseTopicLogicalQueue brokerName={} brokerAddr={} queueId={} logicalQueueIdx={} ok: {}", toBrokerName, toBrokerAddr, queueId, logicalQueueIdx, toQueueRouteData); - break; - } catch (MQBrokerException e) { - if ("queue writable".equals(e.getErrorMessage())) { - log.info("reuseTopicLogicalQueue brokerName={} brokerAddr={} queueId={} logicalQueueIdx={} writable, try again.", toBrokerName, toBrokerAddr, queueId, logicalQueueIdx); - continue RETRY; - } else { - throw e; - } - } - } - } - break; - } - if (toQueueRouteData == null) { - toQueueRouteData = mqAdminExt.createMessageQueueForLogicalQueue(toBrokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.WriteOnly); - log.info("createMessageQueueForLogicalQueue brokerName={} brokerAddr={} logicalQueueIdx={} ok: {}", toBrokerName, toBrokerAddr, logicalQueueIdx, toQueueRouteData); - } - MigrateLogicalQueueBody migrateLogicalQueueBody; - if (forceDelta == null) { - try { - migrateLogicalQueueBody = mqAdminExt.migrateTopicLogicalQueuePrepare(fromQueueRouteData, toQueueRouteData); - } catch (RemotingConnectException e) { - throw new SubCommandException("migrateTopicLogicalQueuePrepare", e); - } - fromQueueRouteData = migrateLogicalQueueBody.getFromQueueRouteData(); - toQueueRouteData = migrateLogicalQueueBody.getToQueueRouteData(); - log.info("migrateTopicLogicalQueuePrepare from {} to {}", fromQueueRouteData, toQueueRouteData); - } else { - toQueueRouteData.setLogicalQueueDelta(forceDelta); - log.warn("migrateTopicLogicalQueuePrepare skip with forceDelta={}", forceDelta); - } - migrateLogicalQueueBody = mqAdminExt.migrateTopicLogicalQueueCommit(fromQueueRouteData, toQueueRouteData); - toQueueRouteData = migrateLogicalQueueBody.getToQueueRouteData(); - log.info("migrateTopicLogicalQueueCommit got: {}", toQueueRouteData); - if (forceDelta == null) { - try { - mqAdminExt.migrateTopicLogicalQueueNotify(fromBrokerAddr, fromQueueRouteData, toQueueRouteData); - } finally { - log.info("migrateTopicLogicalQueueNotify fromBroker {} {}", fromQueueRouteData.getBrokerName(), fromBrokerAddr); - } - } - Collection ignoreBrokerNames = Arrays.asList(fromBrokerName, toBrokerName); - Set brokerNames = queueRouteDataList.stream() - .map(LogicalQueueRouteData::getBrokerName) - .filter(v -> !ignoreBrokerNames.contains(v)) - .map(v -> lookupBrokerMasterAddr(brokerAddrTable, v)) - .collect(Collectors.toSet()); - int i = 1; - for (String brokerName : brokerNames) { - String brokerAddr = null; - try { - brokerAddr = lookupBrokerMasterAddr(brokerAddrTable, brokerName); - mqAdminExt.migrateTopicLogicalQueueNotify(brokerAddr, fromQueueRouteData, toQueueRouteData); - } finally { - log.info("migrateTopicLogicalQueueNotify otherBroker {}({}}) ({}/{})", brokerName, brokerAddr, i, brokerNames.size()); - } - } - } - - @Override public void execute(CommandLine commandLine, Options options, - RPCHook rpcHook) throws SubCommandException { - DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(rpcHook); - mqAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); - - try { - String topic = commandLine.getOptionValue("t").trim(); - String newBrokerName = commandLine.getOptionValue("b").trim(); - int logicalQueueIdx = Integer.parseInt(commandLine.getOptionValue("i").trim()); - Long forceDelta = null; - if (commandLine.hasOption("fd")) { - forceDelta = Long.parseLong(commandLine.getOptionValue("fd").trim()); - } - execute(mqAdminExt, topic, logicalQueueIdx, newBrokerName, forceDelta); - } catch (Exception e) { - throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); - } finally { - mqAdminExt.shutdown(); - } - } - - private static String lookupBrokerMasterAddr(Map brokerAddrTable, String brokerName) { - return ofNullable(brokerAddrTable.get(brokerName)).map(BrokerData::selectBrokerAddr).orElse(null); - } -} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java deleted file mode 100644 index 6ce4bac35b..0000000000 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/QueryTopicLogicalQueueMappingCommand.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.tools.command.logicalqueue; - -import com.alibaba.fastjson.JSON; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSortedMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.SortedMap; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.srvutil.ServerUtil; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; -import org.apache.rocketmq.tools.command.SubCommand; -import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class QueryTopicLogicalQueueMappingCommand implements SubCommand { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); - - @Override public String commandName() { - return "queryTopicLogicalQueueMapping"; - } - - @Override public String commandDesc() { - return "query logical queue mapping info of a topic"; - } - - @Override public Options buildCommandlineOptions(Options options) { - Option opt; - - opt = new Option("t", "topic", true, "topic name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("b", "brokerAddr", true, "broker address."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("c", "clusterName", true, "cluster name."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("m", "merge", false, "merge all brokers' result into one."); - opt.setRequired(false); - options.addOption(opt); - - return options; - } - - @Override public void execute(CommandLine commandLine, Options options, - RPCHook rpcHook) throws SubCommandException { - DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); - defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); - - try { - String topic = commandLine.getOptionValue("t").trim(); - List brokerAddrs; - if (commandLine.hasOption("b")) { - brokerAddrs = Collections.singletonList(commandLine.getOptionValue("c").trim()); - } else if (commandLine.hasOption("c")) { - String clusterName = commandLine.getOptionValue("c").trim(); - brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); - } else { - ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); - return; - } - SortedMap>> result = Maps.newTreeMap(); - for (String brokerAddr : brokerAddrs) { - LogicalQueuesInfo one = defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic); - result.put(brokerAddr, ImmutableSortedMap.copyOf(one)); - } - if (commandLine.hasOption("m")) { - SortedMap> mergedResultMap = Maps.newTreeMap(); - result.values().stream().flatMap(map -> map.values().stream()).flatMap(Collection::stream).forEach(queueRouteData -> { - List sortedQueueRouteDataList = mergedResultMap.computeIfAbsent(queueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayList()); - int idx = Collections.binarySearch(sortedQueueRouteDataList, queueRouteData, - Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta) - .thenComparing(LogicalQueueRouteData::getMessageQueue) - .thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); - if (idx < 0) { - idx = -idx - 1; - } - sortedQueueRouteDataList.add(idx, queueRouteData); - }); - System.out.printf("%s%n", JSON.toJSONString(ImmutableList.copyOf(mergedResultMap.values()))); - } else { - System.out.printf("%s%n", JSON.toJSONString(result)); - } - } catch (Exception e) { - throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); - } finally { - defaultMQAdminExt.shutdown(); - } - } -} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java deleted file mode 100644 index 46276a31cd..0000000000 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueMappingCommand.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.tools.command.logicalqueue; - -import com.google.common.collect.Maps; -import java.util.BitSet; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.apache.rocketmq.srvutil.ServerUtil; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; -import org.apache.rocketmq.tools.command.SubCommand; -import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UpdateTopicLogicalQueueMappingCommand implements SubCommand { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); - - @Override public String commandName() { - return "updateTopicLogicalQueueMapping"; - } - - @Override public String commandDesc() { - return "update logical queue mapping info of a topic"; - } - - @Override public Options buildCommandlineOptions(Options options) { - Option opt; - - opt = new Option("t", "topic", true, "topic name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("q", "queue", true, "message queue id."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("i", "index", true, "logical queue index."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("b", "broker", true, "broker addr."); - opt.setRequired(false); - options.addOption(opt); - - opt = new Option("c", "clusterName", true, "cluster name."); - opt.setRequired(false); - options.addOption(opt); - - return options; - } - - public void execute(DefaultMQAdminExt defaultMQAdminExt, String topic, Collection brokerAddrs) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - Map topicConfigMap = Maps.newHashMapWithExpectedSize(brokerAddrs.size()); - Map allocatedMessageQueueMap = Maps.newHashMap(); - BitSet allocatedLogicalQueueIndices = new BitSet(); - brokerAddrs = brokerAddrs.stream().sorted().collect(Collectors.toList()); - for (String brokerAddr : brokerAddrs) { - TopicConfig topicConfig = defaultMQAdminExt.examineTopicConfig(brokerAddr, topic); - if (topicConfig == null) { - log.warn("examineTopicConfig brokerAddr={} topic={} not exist, skip!", brokerAddr, topic); - continue; - } - topicConfigMap.put(brokerAddr, topicConfig); - - BitSet allocatedQueueIds = new BitSet(); - Optional.ofNullable(defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic)) - .ifPresent(queueRouteData -> queueRouteData.forEach((idx, value) -> { - allocatedLogicalQueueIndices.set(idx); - value.stream().mapToInt(LogicalQueueRouteData::getQueueId).forEach(allocatedQueueIds::set); - })); - allocatedMessageQueueMap.put(brokerAddr, allocatedQueueIds); - } - int unallocatedLogicalQueueIdx = -1; - for (Map.Entry entry : topicConfigMap.entrySet()) { - String brokerAddr = entry.getKey(); - TopicConfig topicConfig = entry.getValue(); - int queueNums = Integer.max(topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums()); - BitSet allocatedQueueIds = allocatedMessageQueueMap.get(brokerAddr); - for (int unallocatedQueueId = allocatedQueueIds.nextClearBit(0); unallocatedQueueId < queueNums; unallocatedQueueId = allocatedQueueIds.nextClearBit(unallocatedQueueId + 1)) { - unallocatedLogicalQueueIdx = allocatedLogicalQueueIndices.nextClearBit(unallocatedLogicalQueueIdx + 1); - log.info("updateTopicLogicalQueueMapping brokerAddr={} topic={} queueId={} to {}", brokerAddr, topic, unallocatedQueueId, unallocatedLogicalQueueIdx); - defaultMQAdminExt.updateTopicLogicalQueueMapping(brokerAddr, topic, unallocatedQueueId, unallocatedLogicalQueueIdx); - allocatedQueueIds.set(unallocatedQueueId); - allocatedLogicalQueueIndices.set(unallocatedLogicalQueueIdx); - } - } - } - - @Override public void execute(CommandLine commandLine, Options options, - RPCHook rpcHook) throws SubCommandException { - DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); - defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); - - try { - defaultMQAdminExt.start(); - String topic = commandLine.getOptionValue("t").trim(); - List brokerAddrs; - if (commandLine.hasOption("b")) { - String brokerAddr = commandLine.getOptionValue("b").trim(); - boolean hasQueueId = commandLine.hasOption("q"); - boolean hasLogicalQueueIndex = commandLine.hasOption("i"); - if (hasQueueId && hasLogicalQueueIndex) { - int queueId = Integer.parseInt(commandLine.getOptionValue("q").trim()); - int logicalQueueIndex = Integer.parseInt(commandLine.getOptionValue("i").trim()); - defaultMQAdminExt.updateTopicLogicalQueueMapping(brokerAddr, topic, queueId, logicalQueueIndex); - log.info("updateTopicLogicalQueueMapping brokerAddr={} topic={} queueId={} to {}", brokerAddr, topic, queueId, logicalQueueIndex); - return; - } else if (hasQueueId || hasLogicalQueueIndex) { - log.error("logicalQueueIndex and queueId must be specified together."); - return; - } else { - log.error("brokerAddr specified but no logicalQueueIndex and queueId found"); - return; - } - } else if (commandLine.hasOption("c")) { - String clusterName = commandLine.getOptionValue("c").trim(); - brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); - } else { - ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); - return; - } - this.execute(defaultMQAdminExt, topic, brokerAddrs); - } catch (Exception e) { - throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); - } finally { - defaultMQAdminExt.shutdown(); - } - } -} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java deleted file mode 100644 index 6b9dd2fdf8..0000000000 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/logicalqueue/UpdateTopicLogicalQueueNumCommand.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.tools.command.logicalqueue; - -import com.google.common.collect.ListMultimap; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Multimaps; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.NavigableMap; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.constant.LoggerName; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.exception.RemotingConnectException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; -import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.apache.rocketmq.tools.command.CommandUtil; -import org.apache.rocketmq.tools.command.SubCommand; -import org.apache.rocketmq.tools.command.SubCommandException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UpdateTopicLogicalQueueNumCommand implements SubCommand { - private static final Logger log = LoggerFactory.getLogger(LoggerName.STDOUT_LOGGER_NAME); - - @Override public String commandName() { - return "updateTopicLogicalQueueNum"; - } - - @Override public String commandDesc() { - return "change logical queue num (increase or decrease) of a topic."; - } - - @Override public Options buildCommandlineOptions(Options options) { - Option opt; - - opt = new Option("t", "topic", true, "topic name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("c", "clusterName", true, "cluster name."); - opt.setRequired(true); - options.addOption(opt); - - opt = new Option("n", "num", true, "logical queue num."); - opt.setRequired(true); - options.addOption(opt); - - return options; - } - - @Override public void execute(CommandLine commandLine, Options options, - RPCHook rpcHook) throws SubCommandException { - DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); - defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); - try { - defaultMQAdminExt.start(); - String clusterName = commandLine.getOptionValue("c").trim(); - String topic = commandLine.getOptionValue("t").trim(); - int newLogicalQueueNum = Integer.parseUnsignedInt(commandLine.getOptionValue("n")); - execute(defaultMQAdminExt, clusterName, topic, newLogicalQueueNum); - } catch (Exception e) { - throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); - } finally { - defaultMQAdminExt.shutdown(); - } - } - - public void execute(DefaultMQAdminExt defaultMQAdminExt, String clusterName, String topic, - int newLogicalQueueNum) throws RemotingSendRequestException, RemotingConnectException, RemotingTimeoutException, MQBrokerException, InterruptedException, SubCommandException { - List brokerAddrs = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName).stream().sorted().collect(Collectors.toList()); - Map topicConfigsByBrokerAddr = Maps.newHashMapWithExpectedSize(brokerAddrs.size()); - NavigableMap> allLogicalQueueMapByIndex = Maps.newTreeMap(); - Map allLogicalQueueMapByBroker = Maps.newHashMap(); - for (String brokerAddr : brokerAddrs) { - TopicConfig topicConfig = defaultMQAdminExt.examineTopicConfig(brokerAddr, topic); - if (topicConfig == null) { - log.info("examineTopicConfig brokerAddr={} topic={} not exist, skip!", brokerAddr, topic); - continue; - } - topicConfigsByBrokerAddr.put(brokerAddr, topicConfig); - - LogicalQueuesInfo logicalQueuesInfo = defaultMQAdminExt.queryTopicLogicalQueueMapping(brokerAddr, topic); - if (logicalQueuesInfo == null) { - throw new SubCommandException(String.format(Locale.ENGLISH, "broker=%s topic=%s logical queue not enabled", brokerAddr, topic)); - } - allLogicalQueueMapByBroker.put(brokerAddr, logicalQueuesInfo); - logicalQueuesInfo.values().stream().flatMap(Collection::stream).forEach(queueRouteData -> { - List sortedQueueRouteDataList = allLogicalQueueMapByIndex.computeIfAbsent(queueRouteData.getLogicalQueueIndex(), ignore -> Lists.newArrayListWithExpectedSize(1)); - int idx = Collections.binarySearch(sortedQueueRouteDataList, queueRouteData, - Comparator.comparingLong(LogicalQueueRouteData::getLogicalQueueDelta) - .thenComparing(LogicalQueueRouteData::getMessageQueue) - .thenComparingInt(LogicalQueueRouteData::getStateOrdinal)); - if (idx < 0) { - idx = -idx - 1; - } - sortedQueueRouteDataList.add(idx, queueRouteData); - }); - } - int oldLogicalQueueNum = (int) allLogicalQueueMapByIndex.values().stream().filter(queueRouteDataList -> queueRouteDataList.stream().anyMatch(LogicalQueueRouteData::isWritable)).count(); - if (oldLogicalQueueNum == newLogicalQueueNum) { - log.info("logical queue num not changed!"); - } else if (oldLogicalQueueNum < newLogicalQueueNum) { - increaseLogicalQueueNum(defaultMQAdminExt, allLogicalQueueMapByBroker, allLogicalQueueMapByIndex, topicConfigsByBrokerAddr, oldLogicalQueueNum, newLogicalQueueNum); - } else { - decreaseLogicalQueueNum(defaultMQAdminExt, allLogicalQueueMapByIndex, oldLogicalQueueNum, newLogicalQueueNum); - } - } - - private void increaseLogicalQueueNum(DefaultMQAdminExt defaultMQAdminExt, - Map allLogicalQueuesInfoMapByBroker, - NavigableMap> allLogicalQueueMapByIndex, - Map topicConfigsByBrokerAddr, int oldLogicalQueueNum, - int newLogicalQueueNum) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { - int curLogicalQueueNum = oldLogicalQueueNum; - String topic = topicConfigsByBrokerAddr.values().stream().findAny().map(TopicConfig::getTopicName).get(); - // try use queue not be assigned as logical queue - for (Map.Entry e : topicConfigsByBrokerAddr.entrySet()) { - String brokerAddr = e.getKey(); - TopicConfig topicConfig = e.getValue(); - LogicalQueuesInfo logicalQueuesInfo = allLogicalQueuesInfoMapByBroker.getOrDefault(brokerAddr, new LogicalQueuesInfo()); - ListMultimap m = Multimaps.index(logicalQueuesInfo.values().stream().flatMap(Collection::stream).iterator(), LogicalQueueRouteData::getQueueId); - for (int queueId = 0, writeQueueNums = topicConfig.getWriteQueueNums(); queueId < writeQueueNums; queueId++) { - if (m.get(queueId).stream().anyMatch(LogicalQueueRouteData::isWritable)) { - continue; - } - int logicalQueueIdx = curLogicalQueueNum; - LogicalQueueRouteData queueRouteData; - try { - queueRouteData = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.Normal); - } finally { - log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); - } - curLogicalQueueNum++; - if (curLogicalQueueNum >= newLogicalQueueNum) { - return; - } - allLogicalQueueMapByIndex.computeIfAbsent(logicalQueueIdx, integer -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); - logicalQueuesInfo.computeIfAbsent(logicalQueueIdx, integer -> Lists.newArrayListWithExpectedSize(1)).add(queueRouteData); - } - } - // try reuse still sealing logical queue - for (Map.Entry> entry : allLogicalQueueMapByIndex.entrySet()) { - List queueRouteDataList = entry.getValue(); - if (queueRouteDataList.size() == 0 || queueRouteDataList.stream().anyMatch(LogicalQueueRouteData::isWritable)) { - continue; - } - int logicalQueueIdx = entry.getKey(); - // this is a sealing logical queue - LogicalQueueRouteData queueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1); - String brokerAddr = queueRouteData.getBrokerAddr(); - List queueRouteDataListByBroker = allLogicalQueuesInfoMapByBroker.get(brokerAddr).get(logicalQueueIdx); - if (queueRouteData.isExpired()) { - int queueId = queueRouteData.getQueueId(); - try { - queueRouteData = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueId, logicalQueueIdx, MessageQueueRouteState.Normal); - } finally { - log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); - } - queueRouteDataList.add(queueRouteData); - queueRouteDataListByBroker.add(queueRouteData); - } else { - // create a message queue in last broker - // not expired message queue can not be reused, since delta value will not be described by one `long` - int queueId = -1; - try { - queueRouteData = defaultMQAdminExt.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.Normal); - queueId = queueRouteData.getQueueId(); - } finally { - log.info("updateTopicLogicalQueueMapping create message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); - } - queueRouteDataList.add(queueRouteData); - queueRouteDataListByBroker.add(queueRouteData); - topicConfigsByBrokerAddr.get(brokerAddr).setWriteQueueNums(queueId + 1); - } - curLogicalQueueNum++; - if (curLogicalQueueNum >= newLogicalQueueNum) { - return; - } - } - // try broker already with expired message queue - for (Map.Entry entry : allLogicalQueuesInfoMapByBroker.entrySet()) { - String brokerAddr = entry.getKey(); - for (Iterator it = entry.getValue().values().stream().flatMap(Collection::stream) - .filter(LogicalQueueRouteData::isExpired) - .sorted(Comparator.comparingInt(LogicalQueueRouteData::getLogicalQueueIndex).thenComparingInt(LogicalQueueRouteData::getQueueId)) - .limit(newLogicalQueueNum - curLogicalQueueNum) - .iterator(); it.hasNext(); ) { - LogicalQueueRouteData queueRouteData = it.next(); - try { - LogicalQueueRouteData result = defaultMQAdminExt.reuseTopicLogicalQueue(brokerAddr, topic, queueRouteData.getQueueId(), queueRouteData.getLogicalQueueIndex(), MessageQueueRouteState.Normal); - // modify in-place - queueRouteData.copyFrom(result); - } finally { - log.info("updateTopicLogicalQueueMapping reuse expired message queue from sealing logical queue brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueRouteData.getQueueId(), queueRouteData.getLogicalQueueIndex()); - } - allLogicalQueueMapByIndex.get(queueRouteData.getLogicalQueueIndex()).stream() - .filter(LogicalQueueRouteData::isExpired) - .filter(v -> Objects.equals(brokerAddr, v.getBrokerAddr()) && queueRouteData.getQueueId() == v.getQueueId() && queueRouteData.getLogicalQueueIndex() == v.getLogicalQueueIndex()) - .forEach(v -> v.copyFrom(queueRouteData)); - curLogicalQueueNum++; - if (curLogicalQueueNum >= newLogicalQueueNum) { - return; - } - } - } - - // try broker with least amount message queue, if amount equal, random select - for (; curLogicalQueueNum < newLogicalQueueNum; curLogicalQueueNum++) { - Map.Entry entry = allLogicalQueuesInfoMapByBroker.entrySet().stream().min(Comparator.comparingInt(value -> value.getValue().values().stream().flatMapToInt(l -> IntStream.of(l.size())).sum())).get(); - String brokerAddr = entry.getKey(); - int logicalQueueIdx = curLogicalQueueNum; - int queueId = -1; - LogicalQueueRouteData queueRouteData; - try { - queueRouteData = defaultMQAdminExt.createMessageQueueForLogicalQueue(brokerAddr, topic, logicalQueueIdx, MessageQueueRouteState.Normal); - queueId = queueRouteData.getQueueId(); - } finally { - log.info("updateTopicLogicalQueueMapping create message queue from fresh brokerAddr={} queueId={} logicalQueueIdx={}", brokerAddr, queueId, logicalQueueIdx); - } - entry.getValue().put(logicalQueueIdx, Lists.newArrayList(queueRouteData)); - } - } - - private void decreaseLogicalQueueNum(DefaultMQAdminExt defaultMQAdminExt, - NavigableMap> allLogicalQueueMapByIndex, - int oldLogicalQueueNum, - int newLogicalQueueNum) throws InterruptedException, RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, MQBrokerException, SubCommandException { - // seal logical queue from greatest index - Map.Entry> maxActiveEntry = allLogicalQueueMapByIndex.lastEntry(); - int curLogicalQueueNum = oldLogicalQueueNum; - while (curLogicalQueueNum > newLogicalQueueNum) { - boolean anyQueueSealed = false; - for (LogicalQueueRouteData queueRouteData : maxActiveEntry.getValue()) { - if (queueRouteData.isWritable()) { - anyQueueSealed = true; - LogicalQueueRouteData resultQueueRouteData = queueRouteData; - try { - resultQueueRouteData = defaultMQAdminExt.sealTopicLogicalQueue(queueRouteData.getBrokerAddr(), queueRouteData); - } finally { - log.info("seal message queue: {}", resultQueueRouteData); - } - } - } - if (anyQueueSealed) { - curLogicalQueueNum--; - } - maxActiveEntry = allLogicalQueueMapByIndex.lowerEntry(maxActiveEntry.getKey()); - if (maxActiveEntry == null) { - throw new SubCommandException(String.format(Locale.ENGLISH, "oldLogicalQueueNum=%d newLogicalQueueNum=%d curLogicalQueueNum=%d but can not find lowerEntry, unexpected situation", oldLogicalQueueNum, newLogicalQueueNum, curLogicalQueueNum)); - } - } - } -} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java new file mode 100644 index 0000000000..ba4e54e111 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.topic; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.rpc.ClientMetadata; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingOne; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.common.statictopic.TopicRemappingDetailWrapper; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminUtils; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class RemappingStaticTopicSubCommand implements SubCommand { + + @Override + public String commandName() { + return "remappingStaticTopic"; + } + + @Override + public String commandDesc() { + return "Update or create static topic, which has fixed number of queues"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + OptionGroup optionGroup = new OptionGroup(); + + Option opt = null; + + opt = new Option("c", "clusters", true, "remapping static topic to clusters, comma separated"); + optionGroup.addOption(opt); + + opt = new Option("b", "brokers", true, "remapping static topic to brokers, comma separated"); + optionGroup.addOption(opt); + + optionGroup.setRequired(true); + options.addOptionGroup(optionGroup); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("mf", "mapFile", true, "The mapping data file name "); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("fr", "forceReplace", true, "Force replace the old mapping"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public void executeFromFile(final CommandLine commandLine, final Options options, + RPCHook rpcHook) throws SubCommandException { + + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String topic = commandLine.getOptionValue('t').trim(); + + String mapFileName = commandLine.getOptionValue('f').trim(); + String mapData = MixAll.file2String(mapFileName); + TopicRemappingDetailWrapper wrapper = TopicRemappingDetailWrapper.decode(mapData.getBytes(), TopicRemappingDetailWrapper.class); + //double check the config + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, wrapper.getBrokerConfigMap()); + TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(wrapper.getBrokerConfigMap().values())), false, true); + + + boolean force = false; + if (commandLine.hasOption("fr") && Boolean.parseBoolean(commandLine.getOptionValue("fr").trim())) { + force = true; + } + MQAdminUtils.remappingStaticTopic(topic, wrapper.getBrokerToMapIn(), wrapper.getBrokerToMapOut(), wrapper.getBrokerConfigMap(), 10000, force, defaultMQAdminExt); + return; + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, + RPCHook rpcHook) throws SubCommandException { + if (!commandLine.hasOption('t')) { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + + if (commandLine.hasOption("f")) { + executeFromFile(commandLine, options, rpcHook); + return; + } + + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + ClientMetadata clientMetadata = new ClientMetadata(); + Map brokerConfigMap = new HashMap<>(); + Map globalIdMap = new HashMap<>(); + Set targetBrokers = new HashSet<>(); + + try { + defaultMQAdminExt.start(); + if (!commandLine.hasOption("b") && !commandLine.hasOption('c')) { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + String topic = commandLine.getOptionValue('t').trim(); + + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + if (clusterInfo == null + || clusterInfo.getClusterAddrTable().isEmpty()) { + throw new RuntimeException("The Cluster info is empty"); + } + clientMetadata.refreshClusterInfo(clusterInfo); + { + if (commandLine.hasOption("b")) { + String brokerStrs = commandLine.getOptionValue("b").trim(); + for (String broker: brokerStrs.split(",")) { + targetBrokers.add(broker.trim()); + } + } else if (commandLine.hasOption("c")) { + String clusters = commandLine.getOptionValue('c').trim(); + for (String cluster : clusters.split(",")) { + cluster = cluster.trim(); + if (clusterInfo.getClusterAddrTable().get(cluster) != null) { + targetBrokers.addAll(clusterInfo.getClusterAddrTable().get(cluster)); + } + } + } + if (targetBrokers.isEmpty()) { + throw new RuntimeException("Find none brokers, do nothing"); + } + for (String broker : targetBrokers) { + String addr = clientMetadata.findMasterBrokerAddr(broker); + if (addr == null) { + throw new RuntimeException("Can't find addr for broker " + broker); + } + } + } + + brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + if (brokerConfigMap.isEmpty()) { + throw new RuntimeException("No topic route to do the remapping"); + } + Map.Entry maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + { + TopicRemappingDetailWrapper oldWrapper = new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_CREATE_OR_UPDATE, maxEpochAndNum.getKey(), brokerConfigMap, new HashSet(), new HashSet()); + String oldMappingDataFile = TopicQueueMappingUtils.writeToTemp(oldWrapper, false); + System.out.printf("The old mapping data is written to file " + oldMappingDataFile + "\n"); + } + + TopicRemappingDetailWrapper newWrapper = TopicQueueMappingUtils.remappingStaticTopic(topic, brokerConfigMap, targetBrokers); + + { + String newMappingDataFile = TopicQueueMappingUtils.writeToTemp(newWrapper, true); + System.out.printf("The old mapping data is written to file " + newMappingDataFile + "\n"); + } + + MQAdminUtils.completeNoTargetBrokers(newWrapper.getBrokerConfigMap(), defaultMQAdminExt); + + MQAdminUtils.remappingStaticTopic(topic, newWrapper.getBrokerToMapIn(), newWrapper.getBrokerToMapOut(), newWrapper.getBrokerConfigMap(), 10000, false, defaultMQAdminExt); + + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java new file mode 100644 index 0000000000..d67b8fbf27 --- /dev/null +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.tools.command.topic; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.body.ClusterInfo; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils; +import org.apache.rocketmq.common.statictopic.TopicRemappingDetailWrapper; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.srvutil.ServerUtil; +import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; +import org.apache.rocketmq.tools.admin.MQAdminUtils; +import org.apache.rocketmq.tools.command.SubCommand; +import org.apache.rocketmq.tools.command.SubCommandException; + +import java.util.AbstractMap; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class UpdateStaticTopicSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateStaticTopic"; + } + + @Override + public String commandDesc() { + return "Update or create static topic, which has fixed number of queues"; + } + + @Override + public Options buildCommandlineOptions(Options options) { + OptionGroup optionGroup = new OptionGroup(); + + Option opt = null; + + opt = new Option("c", "clusters", true, "remapping static topic to clusters, comma separated"); + optionGroup.addOption(opt); + + opt = new Option("b", "brokers", true, "remapping static topic to brokers, comma separated"); + optionGroup.addOption(opt); + + optionGroup.setRequired(true); + options.addOptionGroup(optionGroup); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("qn", "totalQueueNum", true, "total queue num"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("mf", "mapFile", true, "The mapping data file name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("fr", "forceReplace", true, "Force replace the old mapping"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + + public void executeFromFile(final CommandLine commandLine, final Options options, + RPCHook rpcHook) throws SubCommandException { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String topic = commandLine.getOptionValue('t').trim(); + String mapFileName = commandLine.getOptionValue('f').trim(); + String mapData = MixAll.file2String(mapFileName); + TopicRemappingDetailWrapper wrapper = TopicRemappingDetailWrapper.decode(mapData.getBytes(), TopicRemappingDetailWrapper.class); + //double check the config + TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, wrapper.getBrokerConfigMap()); + boolean force = false; + if (commandLine.hasOption("fr") && Boolean.parseBoolean(commandLine.getOptionValue("fr").trim())) { + force = true; + } + TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<>(TopicQueueMappingUtils.getMappingDetailFromConfig(wrapper.getBrokerConfigMap().values())), false, true); + + MQAdminUtils.completeNoTargetBrokers(wrapper.getBrokerConfigMap(), defaultMQAdminExt); + MQAdminUtils.updateTopicConfigMappingAll(wrapper.getBrokerConfigMap(), defaultMQAdminExt, false); + return; + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } + + + + @Override + public void execute(final CommandLine commandLine, final Options options, + RPCHook rpcHook) throws SubCommandException { + if (!commandLine.hasOption('t')) { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + + if (commandLine.hasOption("f")) { + executeFromFile(commandLine, options, rpcHook); + return; + } + + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + Map brokerConfigMap = new HashMap<>(); + Set targetBrokers = new HashSet<>(); + + try { + defaultMQAdminExt.start(); + if ((!commandLine.hasOption("b") && !commandLine.hasOption('c')) + || !commandLine.hasOption("qn")) { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + return; + } + String topic = commandLine.getOptionValue('t').trim(); + + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + if (clusterInfo == null + || clusterInfo.getClusterAddrTable().isEmpty()) { + throw new RuntimeException("The Cluster info is empty"); + } + { + if (commandLine.hasOption("b")) { + String brokerStrs = commandLine.getOptionValue("b").trim(); + for (String broker: brokerStrs.split(",")) { + targetBrokers.add(broker.trim()); + } + } else if (commandLine.hasOption("c")) { + String clusters = commandLine.getOptionValue('c').trim(); + for (String cluster : clusters.split(",")) { + cluster = cluster.trim(); + if (clusterInfo.getClusterAddrTable().get(cluster) != null) { + targetBrokers.addAll(clusterInfo.getClusterAddrTable().get(cluster)); + } + } + } + if (targetBrokers.isEmpty()) { + throw new RuntimeException("Find none brokers, do nothing"); + } + } + + //get the existed topic config and mapping + + brokerConfigMap = MQAdminUtils.examineTopicConfigAll(topic, defaultMQAdminExt); + int queueNum = Integer.parseInt(commandLine.getOptionValue("qn").trim()); + + Map.Entry maxEpochAndNum = new AbstractMap.SimpleImmutableEntry<>(System.currentTimeMillis(), queueNum); + if (!brokerConfigMap.isEmpty()) { + maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap); + } + + { + TopicRemappingDetailWrapper oldWrapper = new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_CREATE_OR_UPDATE, maxEpochAndNum.getKey(), brokerConfigMap, new HashSet(), new HashSet()); + String oldMappingDataFile = TopicQueueMappingUtils.writeToTemp(oldWrapper, false); + System.out.printf("The old mapping data is written to file " + oldMappingDataFile + "\n"); + } + //add the existed brokers to target brokers + targetBrokers.addAll(brokerConfigMap.keySet()); + + //calculate the new data + TopicRemappingDetailWrapper newWrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, targetBrokers, brokerConfigMap); + + { + String newMappingDataFile = TopicQueueMappingUtils.writeToTemp(newWrapper, true); + System.out.printf("The new mapping data is written to file " + newMappingDataFile + "\n"); + } + + MQAdminUtils.completeNoTargetBrokers(brokerConfigMap, defaultMQAdminExt); + MQAdminUtils.updateTopicConfigMappingAll(brokerConfigMap, defaultMQAdminExt, false); + } catch (Exception e) { + throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); + } finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java index b7f5379d36..c33ae52dbd 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -29,7 +29,6 @@ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.CommandUtil; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; -import org.apache.rocketmq.tools.command.logicalqueue.UpdateTopicLogicalQueueMappingCommand; public class UpdateTopicSubCommand implements SubCommand { @@ -68,10 +67,6 @@ public class UpdateTopicSubCommand implements SubCommand { opt.setRequired(false); options.addOption(opt); - opt = new Option("lq", "logicalQueue", true, "set logical queue nums"); - opt.setRequired(false); - options.addOption(opt); - opt = new Option("p", "perm", true, "set topic's permission(2|4|6), intro[2:W 4:R; 6:RW]"); opt.setRequired(false); options.addOption(opt); @@ -137,17 +132,7 @@ public class UpdateTopicSubCommand implements SubCommand { } topicConfig.setOrder(isOrder); - boolean useLogicalQueue = false; - if (commandLine.hasOption("lq")) { - useLogicalQueue = Boolean.parseBoolean(commandLine.getOptionValue("lq").trim()); - } - if (commandLine.hasOption('b')) { - if (useLogicalQueue) { - System.out.printf("-lq and -b can not be used together.%n"); - return; - } - String addr = commandLine.getOptionValue('b').trim(); defaultMQAdminExt.start(); @@ -171,7 +156,6 @@ public class UpdateTopicSubCommand implements SubCommand { Set masterSet = CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); - for (String addr : masterSet) { defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); System.out.printf("create topic to %s success.%n", addr); @@ -193,10 +177,6 @@ public class UpdateTopicSubCommand implements SubCommand { } System.out.printf("%s", topicConfig); - - if (useLogicalQueue) { - new UpdateTopicLogicalQueueMappingCommand().execute(defaultMQAdminExt, topicConfig.getTopicName(), masterSet); - } return; } diff --git a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java index f1f237ddda..4514ef43e5 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtTest.java @@ -62,11 +62,9 @@ import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData; -import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo; -import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingConnectException; @@ -74,7 +72,6 @@ import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.tools.admin.api.MessageTrack; -import org.assertj.core.util.Lists; import org.assertj.core.util.Maps; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -83,12 +80,10 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -150,13 +145,7 @@ public class DefaultMQAdminExtTest { topicRouteData.setBrokerDatas(brokerDatas); topicRouteData.setQueueDatas(new ArrayList()); topicRouteData.setFilterServerTable(new HashMap>()); - LogicalQueuesInfo logicalQueuesInfoinfo = new LogicalQueuesInfo(); - logicalQueuesInfoinfo.put(0, Lists.newArrayList( - new LogicalQueueRouteData(0, 0, new MessageQueue(topic1, broker1Name, 0), MessageQueueRouteState.ReadOnly, 0, 1000, 2000, 3000, broker1Addr), - new LogicalQueueRouteData(0, 1000, new MessageQueue(topic1, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr) - )); - topicRouteData.setLogicalQueuesInfo(logicalQueuesInfoinfo); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), any())).thenReturn(topicRouteData); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(topicRouteData); HashMap result = new HashMap<>(); result.put("id", String.valueOf(MixAll.MASTER_ID)); @@ -258,7 +247,8 @@ public class DefaultMQAdminExtTest { put("topic_test_examine_topicConfig", new TopicConfig("topic_test_examine_topicConfig")); } }); - when(mQClientAPIImpl.getAllTopicConfig(anyString(),anyLong())).thenReturn(topicConfigSerializeWrapper); + //when(mQClientAPIImpl.getAllTopicConfig(anyString(),anyLong())).thenReturn(topicConfigSerializeWrapper); + when(mQClientAPIImpl.getTopicConfig(anyString(), anyString(), anyLong())).thenReturn(new TopicConfigAndQueueMapping(new TopicConfig("topic_test_examine_topicConfig"), null)); } @AfterClass @@ -449,7 +439,7 @@ public class DefaultMQAdminExtTest { @Test public void testMaxOffset() throws Exception { - when(mQClientAPIImpl.getMaxOffset(anyString(), anyString(), anyInt(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(100L); + when(mQClientAPIImpl.getMaxOffset(anyString(), anyString(), anyInt(), anyLong())).thenReturn(100L); assertThat(defaultMQAdminExt.maxOffset(new MessageQueue(topic1, broker1Name, 0))).isEqualTo(100L); } @@ -461,19 +451,8 @@ public class DefaultMQAdminExtTest { assertThat(defaultMQAdminExt.searchOffset(new MessageQueue(topic1, broker1Name, 0), System.currentTimeMillis())).isEqualTo(101L); } - @Test - public void testMaxOffset_LogicalQueue() throws Exception { - when(mQClientAPIImpl.getMaxOffset(eq(broker2Addr), anyString(), anyInt(), anyBoolean(), anyBoolean(), anyLong())).thenReturn(10L); - assertThat(defaultMQAdminExt.maxOffset(new MessageQueue(topic1, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0))).isEqualTo(1010L); - } - @Test - public void testSearchOffset_LogicalQueue() throws Exception { - when(mQClientAPIImpl.searchOffset(eq(broker2Addr), anyString(), anyInt(), anyLong(), anyLong())).thenReturn(11L); - - assertThat(defaultMQAdminExt.searchOffset(new MessageQueue(topic1, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0), System.currentTimeMillis())).isEqualTo(1011L); - } @Test public void testExamineTopicConfig() throws MQBrokerException, RemotingException, InterruptedException { diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java index 11543958c0..25aa3f8def 100644 --- a/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java +++ b/tools/src/test/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommandTest.java @@ -113,7 +113,7 @@ public class ConsumeMessageCommandTest { @Test public void testExecuteDefaultWhenPullMessageByQueueGotException() throws SubCommandException, InterruptedException, RemotingException, MQClientException, MQBrokerException, NoSuchFieldException, IllegalAccessException { DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); - when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(Exception.class); + when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(MQClientException.class); Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); producerField.setAccessible(true); producerField.set(consumeMessageCommand, defaultMQPullConsumer); @@ -135,7 +135,7 @@ public class ConsumeMessageCommandTest { @Test public void testExecuteByConditionWhenPullMessageByQueueGotException() throws IllegalAccessException, InterruptedException, RemotingException, MQClientException, MQBrokerException, NoSuchFieldException, SubCommandException { DefaultMQPullConsumer defaultMQPullConsumer = mock(DefaultMQPullConsumer.class); - when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(Exception.class); + when(defaultMQPullConsumer.pull(any(MessageQueue.class), anyString(), anyLong(), anyInt())).thenThrow(MQClientException.class); Field producerField = ConsumeMessageCommand.class.getDeclaredField("defaultMQPullConsumer"); producerField.setAccessible(true); producerField.set(consumeMessageCommand, defaultMQPullConsumer);