opensource/rocketmq
3
0
Fork 0
mirror of https://github.com/apache/rocketmq.git synced 2026-09-21 05:44:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(all):new feature for static topic

Browse Source
This commit is contained in:
dongeforever
2022-02-08 10:38:41 +08:00
committed by RongtongJin
parent 4506f34e24
commit b6ff649291
138 changed files with 7598 additions and 7295 deletions
Download Patch File Download Diff File Expand all files Collapse all files
broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java
+83 -85
View File
@@ -17,29 +17,6 @@
package org.apache.rocketmq.broker;
import com.google.common.collect.Maps;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.rocketmq.acl.AccessValidator;
import org.apache.rocketmq.broker.client.ClientHousekeepingService;
import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener;
@@ -49,7 +26,6 @@ import org.apache.rocketmq.broker.client.ProducerManager;
import org.apache.rocketmq.broker.client.net.Broker2Client;
import org.apache.rocketmq.broker.client.rebalance.RebalanceLockManager;
import org.apache.rocketmq.broker.dledger.DLedgerRoleChangeHandler;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap;
import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
import org.apache.rocketmq.broker.filtersrv.FilterServerManager;
@@ -80,6 +56,8 @@ import org.apache.rocketmq.broker.processor.SendMessageProcessor;
import org.apache.rocketmq.broker.slave.SlaveSynchronize;
import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
import org.apache.rocketmq.broker.topic.TopicConfigManager;
import org.apache.rocketmq.broker.topic.TopicQueueMappingCleanService;
import org.apache.rocketmq.broker.topic.TopicQueueMappingManager;
import org.apache.rocketmq.broker.transaction.AbstractTransactionalMessageCheckListener;
import org.apache.rocketmq.broker.transaction.TransactionalMessageCheckService;
import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
@@ -87,11 +65,9 @@ import org.apache.rocketmq.broker.transaction.queue.DefaultTransactionalMessageC
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageBridge;
import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageServiceImpl;
import org.apache.rocketmq.broker.util.ServiceProvider;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.Configuration;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.UtilAll;
@@ -99,18 +75,16 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.namesrv.RegisterBrokerResult;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.body.ClusterInfo;
import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo;
import org.apache.rocketmq.common.stats.MomentStatsItem;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.RemotingServer;
import org.apache.rocketmq.remoting.common.TlsMode;
import org.apache.rocketmq.remoting.exception.RemotingConnectException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingServer;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
@@ -128,6 +102,30 @@ import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
import org.apache.rocketmq.store.stats.BrokerStats;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
public class BrokerController {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final InternalLogger LOG_PROTECTION = InternalLoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME);
@@ -179,6 +177,7 @@ public class BrokerController {
private RemotingServer remotingServer;
private RemotingServer fastRemotingServer;
private TopicConfigManager topicConfigManager;
private TopicQueueMappingManager topicQueueMappingManager;
private ExecutorService sendMessageExecutor;
private ExecutorService pullMessageExecutor;
private ExecutorService ackMessageExecutor;
@@ -195,12 +194,13 @@ public class BrokerController {
private InetSocketAddress storeHost;
private BrokerFastFailure brokerFastFailure;
private Configuration configuration;
private TopicQueueMappingCleanService topicQueueMappingCleanService;
private FileWatchService fileWatchService;
private TransactionalMessageCheckService transactionalMessageCheckService;
private TransactionalMessageService transactionalMessageService;
private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener;
private Future<?> slaveSyncFuture;
private Map<Class,AccessValidator> accessValidatorMap = new HashMap<Class, AccessValidator>();
private Map<Class, AccessValidator> accessValidatorMap = new HashMap<Class, AccessValidator>();
private long shouldStartTime;
public BrokerController(
@@ -215,6 +215,7 @@ public class BrokerController {
this.messageStoreConfig = messageStoreConfig;
this.consumerOffsetManager = new ConsumerOffsetManager(this);
this.topicConfigManager = new TopicConfigManager(this);
this.topicQueueMappingManager = new TopicQueueMappingManager(this);
this.pullMessageProcessor = new PullMessageProcessor(this);
this.pullRequestHoldService = new PullRequestHoldService(this);
this.popMessageProcessor = new PopMessageProcessor(this);
@@ -231,7 +232,7 @@ public class BrokerController {
this.clientHousekeepingService = new ClientHousekeepingService(this);
this.broker2Client = new Broker2Client(this);
this.subscriptionGroupManager = new SubscriptionGroupManager(this);
this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig);
this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig, this);
this.filterServerManager = new FilterServerManager(this);
this.assignmentManager = new AssignmentManager(this);
@@ -287,18 +288,20 @@ public class BrokerController {
public boolean initialize() throws CloneNotSupportedException {
boolean result = this.topicConfigManager.load();
result = result && this.topicQueueMappingManager.load();
result = result && this.consumerOffsetManager.load();
result = result && this.subscriptionGroupManager.load();
result = result && this.consumerFilterManager.load();
if (result) {
try {
DefaultMessageStore messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig);
messageStore.registerCleanFileHook(topicConfigManager.getLogicalQueueCleanHook());
this.messageStore = messageStore;
this.messageStore =
new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener,
this.brokerConfig);
if (messageStoreConfig.isEnableDLegerCommitLog()) {
DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore);
((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);
((DLedgerCommitLog) ((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler);
}
this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore);
//load plugin
@@ -342,7 +345,6 @@ public class BrokerController {
this.ackThreadPoolQueue,
new ThreadFactoryImpl("AckMessageThread_"));
this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor(
this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
this.brokerConfig.getProcessReplyMessageThreadPoolNums(),
@@ -485,11 +487,11 @@ public class BrokerController {
this.scheduledExecutorService.scheduleAtFixedRate(() -> {
try {
BrokerController.this.refreshBrokerNameMapping();
BrokerController.this.brokerOuterAPI.refreshMetadata();
} catch (Exception e) {
log.error("ScheduledTask examineBrokerClusterInfo exception", e);
log.error("ScheduledTask refresh metadata exception", e);
}
}, 10, 10, TimeUnit.SECONDS);
}, 1, 5, TimeUnit.SECONDS);
if (!messageStoreConfig.isEnableDLegerCommitLog()) {
if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) {
@@ -513,6 +515,8 @@ public class BrokerController {
}
}
this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this);
if (TlsSystemConfig.tlsMode != TlsMode.DISABLED) {
// Register a listener to reload SslContext
try {
@@ -587,9 +591,9 @@ public class BrokerController {
return;
}
for (AccessValidator accessValidator: accessValidators) {
for (AccessValidator accessValidator : accessValidators) {
final AccessValidator validator = accessValidator;
accessValidatorMap.put(validator.getClass(),validator);
accessValidatorMap.put(validator.getClass(), validator);
this.registerServerRPCHook(new RPCHook() {
@Override
@@ -605,26 +609,17 @@ public class BrokerController {
}
}
private void initialRpcHooks() {
List<RPCHook> rpcHooks = ServiceProvider.load(ServiceProvider.RPC_HOOK_ID, RPCHook.class);
if (rpcHooks == null || rpcHooks.isEmpty()) {
return;
}
for (RPCHook rpcHook: rpcHooks) {
for (RPCHook rpcHook : rpcHooks) {
this.registerServerRPCHook(rpcHook);
}
}
private void refreshBrokerNameMapping() throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
ClusterInfo brokerClusterInfo = this.brokerOuterAPI.getBrokerClusterInfo();
brokerClusterInfo.getBrokerAddrTable().forEach((brokerName, data) -> {
String masterBrokerAddr = data.getBrokerAddrs().get(MixAll.MASTER_ID);
this.brokerName2AddrMap.put(brokerName, masterBrokerAddr);
});
}
public String getBrokerAddrByName(String brokerName) {
return this.brokerName2AddrMap.get(brokerName);
}
@@ -878,6 +873,10 @@ public class BrokerController {
this.fastRemotingServer.shutdown();
}
if (this.topicQueueMappingCleanService != null) {
this.topicQueueMappingCleanService.shutdown();
}
if (this.fileWatchService != null) {
this.fileWatchService.shutdown();
}
@@ -995,6 +994,10 @@ public class BrokerController {
this.fastRemotingServer.start();
}
if (this.topicQueueMappingCleanService != null) {
this.topicQueueMappingCleanService.start();
}
if (this.fileWatchService != null) {
this.fileWatchService.start();
}
@@ -1041,7 +1044,6 @@ public class BrokerController {
this.brokerFastFailure.start();
}
}
public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) {
@@ -1053,7 +1055,7 @@ public class BrokerController {
return;
}
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();
topicConfigSerializeWrapper.setDataVersion(dataVersion);
ConcurrentMap<String, TopicConfig> topicConfigTable = topicConfigList.stream()
@@ -1074,49 +1076,41 @@ public class BrokerController {
.collect(Collectors.toConcurrentMap(TopicConfig::getTopicName, Function.identity()));
topicConfigSerializeWrapper.setTopicConfigTable(topicConfigTable);
String brokerName = this.brokerConfig.getBrokerName();
Map<String, LogicalQueuesInfo> logicalQueuesInfoMap = topicConfigList.stream()
Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = topicConfigList.stream()
.map(TopicConfig::getTopicName)
.map(topicName -> Optional.ofNullable(this.topicConfigManager.selectLogicalQueuesInfo(topicName))
.map(info -> {
info.readLock().lock();
try {
return new AbstractMap.SimpleImmutableEntry<>(topicName, new LogicalQueuesInfoInBroker(info, data -> Objects.equals(data.getBrokerName(), brokerName)));
} finally {
info.readLock().unlock();
}
})
.map(topicName -> Optional.ofNullable(this.topicQueueMappingManager.getTopicQueueMapping(topicName))
.map(info -> new AbstractMap.SimpleImmutableEntry<>(topicName, TopicQueueMappingDetail.cloneAsMappingInfo(info)))
.orElse(null))
.filter(Objects::nonNull)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
if (!logicalQueuesInfoMap.isEmpty()) {
topicConfigSerializeWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap);
if (!topicQueueMappingInfoMap.isEmpty()) {
topicConfigSerializeWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);
}
doRegisterBrokerAll(true, false, topicConfigSerializeWrapper);
}
public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) {
TopicConfigSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildTopicConfigSerializeWrapper();
TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper();
topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion());
topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable());
topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map(
entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission())
|| !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) {
ConcurrentHashMap<String, TopicConfig> topicConfigTable = new ConcurrentHashMap<String, TopicConfig>();
Map<String, LogicalQueuesInfo> logicalQueuesInfoMap = Maps.newHashMapWithExpectedSize(topicConfigWrapper.getTopicConfigTable().size());
for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) {
String topicName = topicConfig.getTopicName();
TopicConfig tmp =
new TopicConfig(topicName, topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(),
this.brokerConfig.getBrokerPermission());
topicConfigTable.put(topicName, tmp);
LogicalQueuesInfoInBroker logicalQueuesInfo = this.topicConfigManager.selectLogicalQueuesInfo(topicName);
if (logicalQueuesInfo != null) {
String brokerName = this.brokerConfig.getBrokerName();
logicalQueuesInfoMap.put(topicName, new LogicalQueuesInfoInBroker(logicalQueuesInfo, data -> Objects.equals(data.getBrokerName(), brokerName)));
}
topicConfigTable.put(topicConfig.getTopicName(), tmp);
}
topicConfigWrapper.setTopicConfigTable(topicConfigTable);
topicConfigWrapper.setLogicalQueuesInfoMap(logicalQueuesInfoMap);
}
if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(),
@@ -1129,7 +1123,7 @@ public class BrokerController {
}
private void doRegisterBrokerAll(boolean checkOrderConfig, boolean oneway,
TopicConfigSerializeWrapper topicConfigWrapper) {
TopicConfigAndMappingSerializeWrapper topicConfigWrapper) {
List<RegisterBrokerResult> registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll(
this.brokerConfig.getBrokerClusterName(),
this.getBrokerAddr(),
@@ -1184,6 +1178,10 @@ public class BrokerController {
this.topicConfigManager = topicConfigManager;
}
public TopicQueueMappingManager getTopicQueueMappingManager() {
return topicQueueMappingManager;
}
public String getHAServerAddr() {
return this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort();
}
@@ -1297,7 +1295,6 @@ public class BrokerController {
this.transactionalMessageCheckListener = transactionalMessageCheckListener;
}
public BlockingQueue<Runnable> getEndTransactionThreadPoolQueue() {
return endTransactionThreadPoolQueue;
@@ -1318,8 +1315,7 @@ public class BrokerController {
public void run() {
try {
BrokerController.this.slaveSynchronize.syncAll();
}
catch (Throwable e) {
} catch (Throwable e) {
log.error("ScheduledTask SlaveSynchronize syncAll error.", e);
}
}
@@ -1365,8 +1361,6 @@ public class BrokerController {
log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId);
}
public void changeToMaster(BrokerRole role) {
if (role == BrokerRole.SLAVE) {
return;
@@ -1435,4 +1429,8 @@ public class BrokerController {
public QueryAssignmentProcessor getQueryAssignmentProcessor() {
return queryAssignmentProcessor;
}
public TopicQueueMappingCleanService getTopicQueueMappingCleanService() {
return topicQueueMappingCleanService;
}
}
broker/src/main/java/org/apache/rocketmq/broker/BrokerPathConfigHelper.java
+4
View File
@@ -35,6 +35,10 @@ public class BrokerPathConfigHelper {
return rootDir + File.separator + "config" + File.separator + "topics.json";
}
public static String getTopicQueueMappingPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "topicQueueMapping.json";
}
public static String getConsumerOffsetPath(final String rootDir) {
return rootDir + File.separator + "config" + File.separator + "consumerOffset.json";
}
broker/src/main/java/org/apache/rocketmq/broker/client/rebalance/RebalanceLockManager.java
+5 -4
View File
@@ -16,16 +16,17 @@
*/
package org.apache.rocketmq.broker.client.rebalance;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.message.MessageQueue;
public class RebalanceLockManager {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.REBALANCE_LOCK_LOGGER_NAME);
broker/src/main/java/org/apache/rocketmq/broker/domain/LogicalQueuesInfoInBroker.java
-116
View File
@@ -1,116 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.domain;
import com.alibaba.fastjson.parser.ParserConfig;
import com.google.common.collect.Maps;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil;
import static java.util.Optional.ofNullable;
public class LogicalQueuesInfoInBroker extends LogicalQueuesInfo {
private final ConcurrentMap<Integer, ConcurrentNavigableMap<Long, LogicalQueueRouteData>> queueId2LogicalQueueMap = Maps.newConcurrentMap();
public LogicalQueuesInfoInBroker() {
}
public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other) {
this(other, null);
}
// deep copy
public LogicalQueuesInfoInBroker(LogicalQueuesInfoInBroker other, Predicate<LogicalQueueRouteData> predicate) {
other.readLock().lock();
try {
for (Entry<Integer, List<LogicalQueueRouteData>> entry : other.entrySet()) {
Stream<LogicalQueueRouteData> stream = entry.getValue().stream();
if (predicate != null) {
stream = stream.filter(predicate);
}
this.put(entry.getKey(), stream.map(LogicalQueueRouteData::new).collect(Collectors.toList()));
}
} finally {
other.readLock().unlock();
}
}
public void updateQueueRouteDataByQueueId(int queueId, LogicalQueueRouteData queueRouteData) {
if (queueRouteData == null) {
return;
}
ConcurrentHashMapUtil.computeIfAbsent(queueId2LogicalQueueMap, queueId, k -> new ConcurrentSkipListMap<>()).put(queueRouteData.getOffsetDelta(), queueRouteData);
}
/**
* find logical queue route data for message queues owned by this broker
*/
public LogicalQueueRouteData queryQueueRouteDataByQueueId(int queueId, long offset) {
ConcurrentNavigableMap<Long, LogicalQueueRouteData> m = this.queueId2LogicalQueueMap.get(queueId);
if (m == null || m.isEmpty()) {
return null;
}
Entry<Long, LogicalQueueRouteData> entry = m.floorEntry(offset);
if (entry == null) {
return null;
}
return entry.getValue();
}
public void deleteQueueRouteData(LogicalQueueRouteData logicalQueueRouteData) {
ConcurrentNavigableMap<Long, LogicalQueueRouteData> m = this.queueId2LogicalQueueMap.get(logicalQueueRouteData.getQueueId());
if (m != null) {
m.remove(logicalQueueRouteData.getOffsetDelta(), logicalQueueRouteData);
}
}
public LogicalQueueRouteData nextAvailableLogicalRouteData(LogicalQueueRouteData queueRouteData,
Predicate<LogicalQueueRouteData> predicate) {
this.readLock().lock();
try {
List<LogicalQueueRouteData> queueRouteDataList = ofNullable(this.get(queueRouteData.getLogicalQueueIndex())).orElse(Collections.emptyList());
int idx = Collections.binarySearch(queueRouteDataList, queueRouteData);
if (idx >= 0) {
for (int i = idx + 1, size = queueRouteDataList.size(); i < size; i++) {
LogicalQueueRouteData tmp = queueRouteDataList.get(i);
if (predicate.test(tmp)) {
return tmp;
}
}
}
} finally {
this.readLock().unlock();
}
return null;
}
static {
// workaround https://github.com/alibaba/fastjson/issues/3730
ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoInBroker.class, GenericMapSuperclassDeserializer.INSTANCE);
}
}
broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java
+44 -13
View File
@@ -16,13 +16,7 @@
*/
package org.apache.rocketmq.broker.out;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.common.DataVersion;
@@ -39,6 +33,7 @@ import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.RegisterBrokerBody;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.QueryDataVersionRequestHeader;
@@ -47,6 +42,9 @@ import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestH
import org.apache.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader;
import org.apache.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.rpc.ClientMetadata;
import org.apache.rocketmq.common.rpc.RpcClient;
import org.apache.rocketmq.common.rpc.RpcClientImpl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.InvokeCallback;
@@ -62,21 +60,39 @@ import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
public class BrokerOuterAPI {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final RemotingClient remotingClient;
private final BrokerController brokerController;
private final TopAddressing topAddressing = new TopAddressing(MixAll.getWSAddr());
private String nameSrvAddr = null;
private final String currBrokerName;
private BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES,
new ArrayBlockingQueue<Runnable>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true));
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) {
this(nettyClientConfig, null);
private ClientMetadata clientMetadata;
private RpcClient rpcClient;
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, final BrokerController brokerController) {
this(nettyClientConfig, null, brokerController, new ClientMetadata());
}
public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) {
private BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook, final BrokerController brokerController, ClientMetadata clientMetadata) {
this.remotingClient = new NettyRemotingClient(nettyClientConfig);
this.clientMetadata = clientMetadata;
this.remotingClient.registerRPCHook(rpcHook);
this.brokerController = brokerController;
this.currBrokerName = brokerController.getBrokerConfig().getBrokerName();
this.rpcClient = new RpcClientImpl(this.clientMetadata, this.remotingClient);
}
public void start() {
@@ -140,7 +156,7 @@ public class BrokerOuterAPI {
requestHeader.setCompressed(compressed);
RegisterBrokerBody requestBody = new RegisterBrokerBody();
requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper);
requestBody.setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper.from(topicConfigWrapper));
requestBody.setFilterServerList(filterServerList);
final byte[] body = requestBody.encode(compressed);
final int bodyCrc32 = UtilAll.crc32(body);
@@ -327,7 +343,7 @@ public class BrokerOuterAPI {
return changedList;
}
public TopicConfigSerializeWrapper getAllTopicConfig(
public TopicConfigAndMappingSerializeWrapper getAllTopicConfig(
final String addr) throws RemotingConnectException, RemotingSendRequestException,
RemotingTimeoutException, InterruptedException, MQBrokerException {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null);
@@ -336,7 +352,7 @@ public class BrokerOuterAPI {
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class);
return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigAndMappingSerializeWrapper.class);
}
default:
break;
@@ -453,4 +469,19 @@ public class BrokerOuterAPI {
public void forwardRequest(String brokerAddr, RemotingCommand request, long timeoutMillis, InvokeCallback invokeCallback) throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException, RemotingTooMuchRequestException, RemotingConnectException {
this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, invokeCallback);
}
public void refreshMetadata() throws Exception {
ClusterInfo brokerClusterInfo = getBrokerClusterInfo();
clientMetadata.refreshClusterInfo(brokerClusterInfo);
}
public ClientMetadata getClientMetadata() {
return clientMetadata;
}
public RpcClient getRpcClient() {
return rpcClient;
}
}
broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java
+1 -173
View File
@@ -16,31 +16,21 @@
*/
package org.apache.rocketmq.broker.processor;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ThreadLocalRandom;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.LongAdder;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
import org.apache.rocketmq.broker.topic.TopicConfigManager;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.TopicQueueId;
import org.apache.rocketmq.common.constant.DBMsgConstants;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
@@ -54,23 +44,17 @@ import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2;
import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.common.sysflag.TopicSysFlag;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.common.utils.ChannelUtil;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.PutMessageResult;
public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor {
protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -80,8 +64,6 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
protected final SocketAddress storeHost;
private List<SendMessageHook> sendMessageHookList;
private final ConcurrentMap<TopicQueueId, LongAdder> inFlyWritingCounterMap = Maps.newConcurrentMap();
public AbstractSendMessageProcessor(final BrokerController brokerController) {
this.brokerController = brokerController;
this.storeHost =
@@ -419,158 +401,4 @@ public abstract class AbstractSendMessageProcessor extends AsyncNettyRequestProc
public boolean rejectRequest() {
return false;
}
public ConcurrentMap<TopicQueueId, LongAdder> getInFlyWritingCounterMap() {
return inFlyWritingCounterMap;
}
protected LogicalQueueContext buildLogicalQueueContext(String topic, int queueId,
RemotingCommand response) {
TopicConfigManager topicConfigManager = this.brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.selectLogicalQueuesInfo(topic);
if (logicalQueuesInfo == null) {
return noopLogicalQueueContext;
}
// writable route data will has largest offset
LogicalQueueRouteData curQueueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, Long.MAX_VALUE);
if (curQueueRouteData == null) {
// topic enabled logical queue, but some message queues are not converted or being converted
String msg = String.format(Locale.ENGLISH, "queueId %d not included in logical queue", queueId);
log.debug("buildLogicalQueueContext unexpected error, topic {} {}", topic, msg);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(msg);
return noopLogicalQueueContext;
}
LongAdder inFlyWritingCounter = ConcurrentHashMapUtil.computeIfAbsent(inFlyWritingCounterMap, new TopicQueueId(topic, queueId), ignore -> new LongAdder());
return new LogicalQueueContext(topic, queueId, logicalQueuesInfo, curQueueRouteData, inFlyWritingCounter);
}
protected class LogicalQueueContext {
private final String topic;
private final int queueId;
private final LogicalQueuesInfoInBroker logicalQueuesInfo;
private final LogicalQueueRouteData curQueueRouteData;
private final LongAdder inFlyWritingCounter;
public LogicalQueueContext(String topic, int queueId,
LogicalQueuesInfoInBroker logicalQueuesInfo,
LogicalQueueRouteData curQueueRouteData, LongAdder inFlyWritingCounter) {
this.topic = topic;
this.queueId = queueId;
this.logicalQueuesInfo = logicalQueuesInfo;
this.curQueueRouteData = curQueueRouteData;
this.inFlyWritingCounter = inFlyWritingCounter;
}
public CompletableFuture<RemotingCommand> hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader,
RemotingCommand request, RemotingCommand response) {
if (curQueueRouteData.isWritable()) {
this.inFlyWritingCounter.increment();
return null;
}
int logicalQueueIdx = curQueueRouteData.getLogicalQueueIndex();
List<LogicalQueueRouteData> queueRouteDataList = logicalQueuesInfo.get(logicalQueueIdx);
LogicalQueueRouteData writableQueueRouteData = null;
for (int i = queueRouteDataList.size() - 1; i >= 0; i--) {
LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i);
if (queueRouteData.isWritable()) {
writableQueueRouteData = queueRouteData;
break;
}
}
if (writableQueueRouteData == null) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark(String.format(Locale.ENGLISH, "broker[%s] topic[%s] queueId[%d] logicalQueueIdx[%d] not writable", AbstractSendMessageProcessor.this.brokerController.getBrokerConfig().getBrokerIP1(), topic, queueId, logicalQueueIdx));
return CompletableFuture.completedFuture(response);
}
if ((Optional.ofNullable(requestHeader.getSysFlag()).orElse(0) & MessageSysFlag.LOGICAL_QUEUE_FLAG) > 0) {
// new client, use redirect
response.setCode(ResponseCode.NO_PERMISSION);
response.addExtField(MessageConst.PROPERTY_REDIRECT, "1");
response.setBody(RemotingSerializable.encode(ImmutableList.of(curQueueRouteData, writableQueueRouteData)));
return CompletableFuture.completedFuture(response);
} else {
// old client, use forward
this.logicalQueueHookForward(ctx, writableQueueRouteData, requestHeader, request, response);
}
if (response.getCode() != -1) {
return CompletableFuture.completedFuture(response);
} else if (response.getCode() == ResponseCode.ASYNC_AND_RETURN_NULL) {
return CompletableFuture.completedFuture(null);
}
return null;
}
private void logicalQueueHookForward(ChannelHandlerContext ctx,
LogicalQueueRouteData writableQueueRouteData,
SendMessageRequestHeader requestHeader, RemotingCommand request,
RemotingCommand response) {
response.setCode(ResponseCode.SUCCESS);
requestHeader.setQueueId(writableQueueRouteData.getQueueId());
request.writeCustomHeader(requestHeader);
String brokerName = writableQueueRouteData.getBrokerName();
BrokerController brokerController = AbstractSendMessageProcessor.this.brokerController;
String brokerAddr = brokerController.getBrokerAddrByName(brokerName);
if (brokerAddr == null) {
log.warn("getBrokerAddrByName brokerName={} got null, fallback to queueRouteData.getBrokerAddr()", brokerName);
brokerAddr = writableQueueRouteData.getBrokerAddr();
}
if (brokerAddr == null) {
response.setCode(ResponseCode.SYSTEM_ERROR);
String msg = String.format(Locale.ENGLISH, "unknown brokerName %s", brokerName);
response.setRemark(msg);
log.warn("logicalQueueHookForward can not look up brokerName={}: {}", brokerName, requestHeader);
return;
}
try {
String finalBrokerAddr = brokerAddr;
brokerController.getBrokerOuterAPI().forwardRequest(brokerAddr, request, brokerController.getBrokerConfig().getForwardTimeout(), responseFuture -> {
RemotingCommand forwardResponse = responseFuture.getResponseCommand();
if (forwardResponse == null) {
forwardResponse = response;
forwardResponse.setCode(ResponseCode.SYSTEM_ERROR);
if (!responseFuture.isSendRequestOK()) {
forwardResponse.setRemark(String.format(Locale.ENGLISH, "send request failed to %s: %s", finalBrokerAddr, responseFuture.getCause()));
} else if (responseFuture.isTimeout()) {
forwardResponse.setRemark(String.format(Locale.ENGLISH, "wait response from %s timeout: %dms", finalBrokerAddr, responseFuture.getTimeoutMillis()));
} else {
forwardResponse.setRemark(String.format(Locale.ENGLISH, "unknown reason. addr: %s, timeoutMillis: %d: %s", finalBrokerAddr, responseFuture.getTimeoutMillis(), responseFuture.getCause()));
}
} else {
CommandCustomHeader customHeader = forwardResponse.readCustomHeader();
if (customHeader instanceof SendMessageResponseHeader) {
SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) customHeader;
Integer forwardQueueId = responseHeader.getQueueId();
forwardResponse.addExtField(MessageConst.PROPERTY_FORWARD_QUEUE_ID, forwardQueueId != null ? Integer.toString(forwardQueueId) : "null");
responseHeader.setQueueId(requestHeader.getQueueId());
// queueOffset should not be changed since forwarded broker will add delta to it.
}
}
AbstractSendMessageProcessor.this.doResponse(ctx, request, forwardResponse);
});
response.setCode(ResponseCode.ASYNC_AND_RETURN_NULL);
} catch (Exception e) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("forward error");
log.warn(String.format(Locale.ENGLISH, "logicalQueueHookForward to %s error", brokerAddr), e);
}
}
public void hookAfterPut(CompletableFuture<PutMessageResult> putMessageResult) {
Optional.ofNullable(putMessageResult).orElse(CompletableFuture.completedFuture(null)).whenComplete((result, throwable) -> {
this.inFlyWritingCounter.decrement();
});
}
}
private final LogicalQueueContext noopLogicalQueueContext = new LogicalQueueContext(null, 0, null, null, null) {
@Override public CompletableFuture<RemotingCommand> hookBeforePut(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader,
RemotingCommand request, RemotingCommand response) {
return null;
}
@Override public void hookAfterPut(CompletableFuture<PutMessageResult> putMessageResult) {
}
};
}
broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+330 -683
View File
File diff suppressed because it is too large Load Diff
broker/src/main/java/org/apache/rocketmq/broker/processor/ConsumerManageProcessor.java
+98 -3
View File
@@ -17,11 +17,9 @@
package org.apache.rocketmq.broker.processor;
import io.netty.channel.ChannelHandlerContext;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
@@ -31,6 +29,12 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHead
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader;
import org.apache.rocketmq.common.rpc.RpcRequest;
import org.apache.rocketmq.common.rpc.RpcResponse;
import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -38,6 +42,10 @@ import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import java.util.List;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -109,6 +117,12 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen
final UpdateConsumerOffsetRequestHeader requestHeader =
(UpdateConsumerOffsetRequestHeader) request
.decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext);
if (rewriteResult != null) {
return rewriteResult;
}
this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getConsumerGroup(),
requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
response.setCode(ResponseCode.SUCCESS);
@@ -116,6 +130,77 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen
return response;
}
public RemotingCommand rewriteRequestForStaticTopic(QueryConsumerOffsetRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {
try {
if (mappingContext.getMappingDetail() == null) {
return null;
}
TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
if (!mappingContext.isLeader()) {
return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));
}
List<LogicQueueMappingItem> mappingItemList = mappingContext.getMappingItemList();
if (mappingItemList.size() == 1
&& mappingItemList.get(0).getLogicOffset() == 0) {
//as physical, just let it go
requestHeader.setQueueId(mappingContext.getLeaderItem().getQueueId());
return null;
}
//double read check
List<LogicQueueMappingItem> itemList = mappingContext.getMappingItemList();
//by default, it is -1
long offset = -1;
//double read, first from leader, then from second leader
for (int i = itemList.size() - 1; i >= 0; i--) {
LogicQueueMappingItem mappingItem = itemList.get(i);
if (mappingItem.getBname().equals(mappingDetail.getBname())) {
offset = this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getConsumerGroup(), requestHeader.getTopic(), mappingItem.getQueueId());
if (offset >= 0) {
break;
} else {
//not found
continue;
}
} else {
//maybe we need to reconstruct an object
requestHeader.setBname(mappingItem.getBname());
requestHeader.setQueueId(mappingItem.getQueueId());
requestHeader.setLo(false);
requestHeader.setSetZeroIfNotFound(false);
RpcRequest rpcRequest = new RpcRequest(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader, null);
RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
if (rpcResponse.getException() != null) {
throw rpcResponse.getException();
}
if (rpcResponse.getCode() == ResponseCode.SUCCESS) {
offset = ((QueryConsumerOffsetResponseHeader) rpcResponse.getHeader()).getOffset();
break;
} else if (rpcResponse.getCode() == ResponseCode.QUERY_NOT_FOUND) {
continue;
} else {
//this should not happen
throw new RuntimeException("Unknown response code " + rpcResponse.getCode());
}
}
}
final RemotingCommand response = RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class);
final QueryConsumerOffsetResponseHeader responseHeader = (QueryConsumerOffsetResponseHeader) response.readCustomHeader();
if (offset >= 0) {
responseHeader.setOffset(offset);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
} else {
response.setCode(ResponseCode.QUERY_NOT_FOUND);
response.setRemark("Not found, maybe this group consumer boot first");
}
return response;
} catch (Throwable t) {
t.printStackTrace();
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
}
private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response =
@@ -126,6 +211,13 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen
(QueryConsumerOffsetRequestHeader) request
.decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class);
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
if (rewriteResult != null) {
return rewriteResult;
}
long offset =
this.brokerController.getConsumerOffsetManager().queryOffset(
requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId());
@@ -138,7 +230,10 @@ public class ConsumerManageProcessor extends AsyncNettyRequestProcessor implemen
long minOffset =
this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(),
requestHeader.getQueueId());
if (minOffset <= 0
if (requestHeader.getSetZeroIfNotFound() != null && Boolean.FALSE.equals(requestHeader.getSetZeroIfNotFound())) {
response.setCode(ResponseCode.QUERY_NOT_FOUND);
response.setRemark("Not found, do not set to zero, maybe this group boot first");
} else if (minOffset <= 0
&& !this.brokerController.getMessageStore().checkInDiskByConsumeOffset(
requestHeader.getTopic(), requestHeader.getQueueId(), 0)) {
responseHeader.setOffset(0L);
broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+253 -116
View File
@@ -16,17 +16,13 @@
*/
package org.apache.rocketmq.broker.processor;
import com.google.common.collect.ImmutableList;
import io.netty.channel.Channel;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.FileRegion;
import java.nio.ByteBuffer;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.filter.ConsumerFilterData;
import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
import org.apache.rocketmq.broker.filter.ExpressionForRetryMessageFilter;
@@ -43,17 +39,22 @@ import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.filter.ExpressionType;
import org.apache.rocketmq.common.filter.FilterAPI;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.protocol.topic.OffsetMovedEvent;
import org.apache.rocketmq.common.rpc.RpcClientUtils;
import org.apache.rocketmq.common.rpc.RpcRequest;
import org.apache.rocketmq.common.rpc.RpcResponse;
import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.common.sysflag.PullSysFlag;
@@ -67,7 +68,6 @@ import org.apache.rocketmq.remoting.netty.AsyncNettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
import org.apache.rocketmq.remoting.netty.RequestTask;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.MessageExtBrokerInner;
import org.apache.rocketmq.store.MessageFilter;
@@ -75,6 +75,11 @@ import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import java.nio.ByteBuffer;
import java.util.List;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
public class PullMessageProcessor extends AsyncNettyRequestProcessor implements NettyRequestProcessor {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController;
@@ -95,6 +100,200 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
return false;
}
private RemotingCommand rewriteRequestForStaticTopic(PullMessageRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {
try {
if (mappingContext.getMappingDetail() == null) {
return null;
}
TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
String topic = mappingContext.getTopic();
Integer globalId = mappingContext.getGlobalId();
// if the leader? consider the order consumer, which will lock the mq
if (!mappingContext.isLeader()) {
return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d cannot find mapping item in request process of current broker %s", topic, globalId, mappingDetail.getBname()));
}
Long globalOffset = requestHeader.getQueueOffset();
LogicQueueMappingItem mappingItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), globalOffset, true);
mappingContext.setCurrentItem(mappingItem);
if (globalOffset < mappingItem.getLogicOffset()) {
//handleOffsetMoved
//If the physical queue is reused, we should handle the PULL_OFFSET_MOVED independently
//Otherwise, we could just transfer it to the physical process
}
//below are physical info
String bname = mappingItem.getBname();
Integer phyQueueId = mappingItem.getQueueId();
Long phyQueueOffset = mappingItem.computePhysicalQueueOffset(globalOffset);
requestHeader.setQueueId(phyQueueId);
requestHeader.setQueueOffset(phyQueueOffset);
if (mappingItem.checkIfEndOffsetDecided()
&& requestHeader.getMaxMsgNums() != null) {
requestHeader.setMaxMsgNums((int) Math.min(mappingItem.getEndOffset() - mappingItem.getStartOffset(), requestHeader.getMaxMsgNums()));
}
if (mappingDetail.getBname().equals(bname)) {
//just let it go, do the local pull process
return null;
}
int sysFlag = requestHeader.getSysFlag();
requestHeader.setLo(false);
requestHeader.setBname(bname);
sysFlag = PullSysFlag.clearSuspendFlag(sysFlag);
sysFlag = PullSysFlag.clearCommitOffsetFlag(sysFlag);
requestHeader.setSysFlag(sysFlag);
RpcRequest rpcRequest = new RpcRequest(RequestCode.PULL_MESSAGE, requestHeader, null);
RpcResponse rpcResponse = this.brokerController.getBrokerOuterAPI().getRpcClient().invoke(rpcRequest, this.brokerController.getBrokerConfig().getForwardTimeout()).get();
if (rpcResponse.getException() != null) {
throw rpcResponse.getException();
}
PullMessageResponseHeader responseHeader = (PullMessageResponseHeader) rpcResponse.getHeader();
{
RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, rpcResponse.getCode());
if (rewriteResult != null) {
return rewriteResult;
}
}
return RpcClientUtils.createCommandForRpcResponse(rpcResponse);
} catch (Throwable t) {
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
}
private RemotingCommand rewriteResponseForStaticTopic(PullMessageRequestHeader requestHeader, PullMessageResponseHeader responseHeader,
TopicQueueMappingContext mappingContext, final int code) {
try {
if (mappingContext.getMappingDetail() == null) {
return null;
}
TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
LogicQueueMappingItem leaderItem = mappingContext.getLeaderItem();
LogicQueueMappingItem currentItem = mappingContext.getCurrentItem();
LogicQueueMappingItem earlistItem = TopicQueueMappingUtils.findLogicQueueMappingItem(mappingContext.getMappingItemList(), 0L, true);
assert currentItem.getLogicOffset() >= 0;
long requestOffset = requestHeader.getQueueOffset();
long nextBeginOffset = responseHeader.getNextBeginOffset();
long minOffset = responseHeader.getMinOffset();
long maxOffset = responseHeader.getMaxOffset();
int responseCode = code;
//consider the following situations
// 1. read from slave, currently not supported
// 2. the middle queue is truncated because of deleting commitlog
if (code != ResponseCode.SUCCESS) {
//note the currentItem maybe both the leader and the earliest
boolean isRevised = false;
if (leaderItem.getGen() == currentItem.getGen()) {
//read the leader
if (requestOffset > maxOffset) {
//actually, we need do nothing, but keep the code structure here
if (code == ResponseCode.PULL_OFFSET_MOVED) {
responseCode = ResponseCode.PULL_OFFSET_MOVED;
nextBeginOffset = maxOffset;
} else {
//maybe current broker is the slave
responseCode = code;
}
} else if (requestOffset < minOffset) {
nextBeginOffset = minOffset;
responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
} else {
responseCode = code;
}
}
//note the currentItem maybe both the leader and the earliest
if (earlistItem.getGen() == currentItem.getGen()) {
//read the earliest one
if (requestOffset < minOffset) {
if (code == ResponseCode.PULL_OFFSET_MOVED) {
responseCode = ResponseCode.PULL_OFFSET_MOVED;
nextBeginOffset = minOffset;
} else {
//maybe read from slave, but we still set it to moved
responseCode = ResponseCode.PULL_OFFSET_MOVED;
nextBeginOffset = minOffset;
}
} else if (requestOffset >= maxOffset) {
//just move to another item
LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
if (nextItem != null) {
isRevised = true;
currentItem = nextItem;
nextBeginOffset = currentItem.getStartOffset();
minOffset = currentItem.getStartOffset();
maxOffset = minOffset;
responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
} else {
//maybe the next one's logic offset is -1
responseCode = ResponseCode.PULL_NOT_FOUND;
}
} else {
//let it go
responseCode = code;
}
}
//read from the middle item, ignore the PULL_OFFSET_MOVED
if (!isRevised
&& leaderItem.getGen() != currentItem.getGen()
&& earlistItem.getGen() != currentItem.getGen()) {
if (requestOffset < minOffset) {
nextBeginOffset = minOffset;
responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
} else if (requestOffset >= maxOffset) {
//just move to another item
LogicQueueMappingItem nextItem = TopicQueueMappingUtils.findNext(mappingContext.getMappingItemList(), currentItem, true);
if (nextItem != null) {
currentItem = nextItem;
nextBeginOffset = currentItem.getStartOffset();
minOffset = currentItem.getStartOffset();
maxOffset = minOffset;
responseCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
} else {
//maybe the next one's logic offset is -1
responseCode = ResponseCode.PULL_NOT_FOUND;
}
} else {
responseCode = code;
}
}
}
//handle nextBeginOffset
//the next begin offset should no more than the end offset
if (currentItem.checkIfEndOffsetDecided()
&& nextBeginOffset >= currentItem.getEndOffset()) {
nextBeginOffset = currentItem.getEndOffset();
}
responseHeader.setNextBeginOffset(currentItem.computeStaticQueueOffsetStrictly(nextBeginOffset));
//handle min offset
responseHeader.setMinOffset(currentItem.computeStaticQueueOffsetStrictly(Math.max(currentItem.getStartOffset(), minOffset)));
//handle max offset
responseHeader.setMaxOffset(Math.max(currentItem.computeStaticQueueOffsetStrictly(maxOffset),
TopicQueueMappingDetail.computeMaxOffsetFromMapping(mappingDetail, mappingContext.getGlobalId())));
//set the offsetDelta
responseHeader.setOffsetDelta(currentItem.computeOffsetDelta());
if (code != ResponseCode.SUCCESS) {
return RemotingCommand.createResponseCommandWithHeader(responseCode, responseHeader);
} else {
return null;
}
} catch (Throwable t) {
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
}
private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend)
throws RemotingCommandException {
RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
@@ -132,25 +331,33 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0;
String topic = requestHeader.getTopic();
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic());
if (null == topicConfig) {
log.error("the topic {} not exist, consumer: {}", topic, RemotingHelper.parseChannelRemoteAddr(channel));
log.error("the topic {} not exist, consumer: {}", requestHeader.getTopic(), RemotingHelper.parseChannelRemoteAddr(channel));
response.setCode(ResponseCode.TOPIC_NOT_EXIST);
response.setRemark(String.format("topic[%s] not exist, apply first please! %s", topic, FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
response.setRemark(String.format("topic[%s] not exist, apply first please! %s", requestHeader.getTopic(), FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)));
return response;
}
if (!PermName.isReadable(topicConfig.getPerm())) {
response.setCode(ResponseCode.NO_PERMISSION);
response.setRemark("the topic[" + topic + "] pulling message is forbidden");
response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden");
return response;
}
int queueId = requestHeader.getQueueId();
if (queueId < 0 || queueId >= topicConfig.getReadQueueNums()) {
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, false);
{
RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
if (rewriteResult != null) {
return rewriteResult;
}
}
if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) {
String errorInfo = String.format("queueId[%d] is illegal, topic:[%s] topicConfig.readQueueNums:[%d] consumer:[%s]",
queueId, topic, topicConfig.getReadQueueNums(), channel.remoteAddress());
requestHeader.getQueueId(), requestHeader.getTopic(), topicConfig.getReadQueueNums(), channel.remoteAddress());
log.warn(errorInfo);
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(errorInfo);
@@ -162,11 +369,11 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
if (hasSubscriptionFlag) {
try {
subscriptionData = FilterAPI.build(
topic, requestHeader.getSubscription(), requestHeader.getExpressionType()
requestHeader.getTopic(), requestHeader.getSubscription(), requestHeader.getExpressionType()
);
if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
consumerFilterData = ConsumerFilterManager.build(
topic, requestHeader.getConsumerGroup(), requestHeader.getSubscription(),
requestHeader.getTopic(), requestHeader.getConsumerGroup(), requestHeader.getSubscription(),
requestHeader.getExpressionType(), requestHeader.getSubVersion()
);
assert consumerFilterData != null;
@@ -195,9 +402,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
return response;
}
subscriptionData = consumerGroupInfo.findSubscriptionData(topic);
subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic());
if (null == subscriptionData) {
log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), topic);
log.warn("the consumer's subscription not exist, group: {}, topic:{}", requestHeader.getConsumerGroup(), requestHeader.getTopic());
response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
response.setRemark("the consumer's subscription not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
return response;
@@ -211,7 +418,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
return response;
}
if (!ExpressionType.isTagType(subscriptionData.getExpressionType())) {
consumerFilterData = this.brokerController.getConsumerFilterManager().get(topic,
consumerFilterData = this.brokerController.getConsumerFilterManager().get(requestHeader.getTopic(),
requestHeader.getConsumerGroup());
if (consumerFilterData == null) {
response.setCode(ResponseCode.FILTER_DATA_NOT_EXIST);
@@ -220,7 +427,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
}
if (consumerFilterData.getClientVersion() < requestHeader.getSubVersion()) {
log.warn("The broker's consumer filter data is not latest, group: {}, topic: {}, serverV: {}, clientV: {}",
requestHeader.getConsumerGroup(), topic, consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
requestHeader.getConsumerGroup(), requestHeader.getTopic(), consumerFilterData.getClientVersion(), requestHeader.getSubVersion());
response.setCode(ResponseCode.FILTER_DATA_NOT_LATEST);
response.setRemark("the consumer's consumer filter data not latest");
return response;
@@ -235,6 +442,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
return response;
}
MessageFilter messageFilter;
if (this.brokerController.getBrokerConfig().isFilterSupportRetry()) {
messageFilter = new ExpressionForRetryMessageFilter(subscriptionData, consumerFilterData,
@@ -244,72 +452,12 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
this.brokerController.getConsumerFilterManager());
}
long offset = requestHeader.getQueueOffset();
int maxMsgNums = requestHeader.getMaxMsgNums();
LogicalQueuesInfoInBroker logicalQueuesInfo = this.brokerController.getTopicConfigManager().selectLogicalQueuesInfo(topic);
LogicalQueueRouteData queueRouteData = null;
if (logicalQueuesInfo != null) {
int responseErrorCode = ResponseCode.SUCCESS;
queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, offset);
if (queueRouteData != null) {
if (queueRouteData.isWriteOnly()) {
responseErrorCode = ResponseCode.PULL_NOT_FOUND;
response.setRemark("logical queue write only");
} else if (queueRouteData.isExpired()) {
responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
response.setRemark("logical queue expired");
prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData);
} else if (MessageQueueRouteState.ReadOnly.equals(queueRouteData.getState()) && queueRouteData.getOffsetMax() >= 0) {
if (offset >= queueRouteData.getOffsetMax()) {
responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
response.setRemark("queue offset exceed offsetMax");
prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData);
} else if (offset + maxMsgNums > queueRouteData.getOffsetMax()) {
if ((queueRouteData.getOffsetMax() - 1 <= this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId)) &&
(this.brokerController.getMessageStore().getCommitLogOffsetInQueue(topic, queueId, queueRouteData.getOffsetMax() - 1) < this.brokerController.getMessageStore().getMinPhyOffset())) {
responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
response.setRemark("queue offset removed");
prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData);
} else {
maxMsgNums = (int) (queueRouteData.getOffsetMax() - offset);
if (maxMsgNums <= 0) {
responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
response.setRemark("queue offset out of range");
prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData);
}
}
}
}
} else {
responseErrorCode = ResponseCode.PULL_RETRY_IMMEDIATELY;
response.setRemark("no suitable queue");
response.addExtField(MessageConst.PROPERTY_REDIRECT, "1");
// instruct client to refresh all
response.setBody(null);
queueRouteData = logicalQueuesInfo.queryQueueRouteDataByQueueId(queueId, 0L);
}
if (responseErrorCode != ResponseCode.SUCCESS) {
response.setCode(responseErrorCode);
responseHeader.setMinOffset(offset);
responseHeader.setMaxOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset);
responseHeader.setNextBeginOffset(queueRouteData != null ? queueRouteData.getOffsetMax() : offset);
responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID);
return response;
}
}
final GetMessageResult getMessageResult =
this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), topic,
queueId, offset, maxMsgNums, messageFilter);
this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), requestHeader.getTopic(),
requestHeader.getQueueId(), requestHeader.getQueueOffset(), requestHeader.getMaxMsgNums(), messageFilter);
if (getMessageResult != null) {
response.setRemark(getMessageResult.getStatus().name());
long nextBeginOffset = getMessageResult.getNextBeginOffset();
if (queueRouteData != null && queueRouteData.getOffsetMax() >= 0 && nextBeginOffset > queueRouteData.getOffsetMax()) {
// prevent from pulling messages from next logical queue route data
nextBeginOffset = queueRouteData.getOffsetMax();
}
responseHeader.setNextBeginOffset(nextBeginOffset);
responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset());
responseHeader.setMinOffset(getMessageResult.getMinOffset());
// this does not need to be modified since it's not an accurate value under logical queue.
responseHeader.setMaxOffset(getMessageResult.getMaxOffset());
@@ -360,9 +508,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
// XXX: warn and notify me
log.info("the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",
requestHeader.getQueueOffset(),
nextBeginOffset,
topic,
queueId,
getMessageResult.getNextBeginOffset(),
requestHeader.getTopic(),
requestHeader.getQueueId(),
requestHeader.getConsumerGroup()
);
} else {
@@ -387,7 +535,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
case OFFSET_TOO_SMALL:
response.setCode(ResponseCode.PULL_OFFSET_MOVED);
log.info("the request offset too small. group={}, topic={}, requestOffset={}, brokerMinOffset={}, clientIp={}",
requestHeader.getConsumerGroup(), topic, requestHeader.getQueueOffset(),
requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueOffset(),
getMessageResult.getMinOffset(), channel.remoteAddress());
break;
default:
@@ -395,11 +543,14 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
break;
}
if (this.hasConsumeMessageHook()) {
ConsumeMessageContext context = new ConsumeMessageContext();
context.setConsumerGroup(requestHeader.getConsumerGroup());
context.setTopic(topic);
context.setQueueId(queueId);
context.setTopic(requestHeader.getTopic());
context.setQueueId(requestHeader.getQueueId());
String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER);
@@ -437,6 +588,12 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
this.executeConsumeMessageHookBefore(context);
}
//rewrite the response for the
RemotingCommand rewriteResult = rewriteResponseForStaticTopic(requestHeader, responseHeader, mappingContext, response.getCode());
if (rewriteResult != null) {
response = rewriteResult;
}
switch (response.getCode()) {
case ResponseCode.SUCCESS:
@@ -483,6 +640,9 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills();
}
String topic = requestHeader.getTopic();
long offset = requestHeader.getQueueOffset();
int queueId = requestHeader.getQueueId();
PullRequest pullRequest = new PullRequest(request, channel, pollingTimeMills,
this.brokerController.getMessageStore().now(), offset, subscriptionData, messageFilter);
this.brokerController.getPullRequestHoldService().suspendPullRequest(topic, queueId, pullRequest);
@@ -490,20 +650,6 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
break;
}
if (queueRouteData != null) {
logicalQueuesInfo.readLock().lock();
try {
List<LogicalQueueRouteData> queueRouteDataList = logicalQueuesInfo.get(queueRouteData.getLogicalQueueIndex());
MessageQueue latestMessageQueue = queueRouteDataList.get(queueRouteDataList.size() - 1).getMessageQueue();
if (!latestMessageQueue.getBrokerName().equals(brokerController.getBrokerConfig().getBrokerName()) || latestMessageQueue.getQueueId() != queueId) {
// There are other newer message queue, instruct client to refresh meta-data to access these
prepareRedirectResponse(response, logicalQueuesInfo, queueRouteData);
}
} finally {
logicalQueuesInfo.readLock().unlock();
}
}
case ResponseCode.PULL_RETRY_IMMEDIATELY:
break;
case ResponseCode.PULL_OFFSET_MOVED:
@@ -518,7 +664,7 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
event.setConsumerGroup(requestHeader.getConsumerGroup());
event.setMessageQueue(mq);
event.setOffsetRequest(requestHeader.getQueueOffset());
event.setOffsetNew(nextBeginOffset);
event.setOffsetNew(getMessageResult.getNextBeginOffset());
this.generateOffsetMovedEvent(event);
log.warn(
"PULL_OFFSET_MOVED:correction offset. topic={}, groupId={}, requestOffset={}, newOffset={}, suggestBrokerId={}",
@@ -547,20 +693,11 @@ public class PullMessageProcessor extends AsyncNettyRequestProcessor implements
&& this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE;
if (storeOffsetEnable) {
this.brokerController.getConsumerOffsetManager().commitOffset(RemotingHelper.parseChannelRemoteAddr(channel),
requestHeader.getConsumerGroup(), topic, queueId, requestHeader.getCommitOffset());
requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset());
}
return response;
}
private void prepareRedirectResponse(RemotingCommand response, LogicalQueuesInfoInBroker logicalQueuesInfo,
LogicalQueueRouteData queueRouteData) {
LogicalQueueRouteData nextReadableLogicalQueueRouteData = logicalQueuesInfo.nextAvailableLogicalRouteData(queueRouteData, LogicalQueueRouteData::isReadable);
if (nextReadableLogicalQueueRouteData != null) {
response.addExtField(MessageConst.PROPERTY_REDIRECT, "1");
response.setBody(RemotingSerializable.encode(ImmutableList.of(queueRouteData, nextReadableLogicalQueueRouteData)));
}
}
public boolean hasConsumeMessageHook() {
return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty();
}
broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java
+73 -28
View File
@@ -28,10 +28,13 @@ import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
import org.apache.rocketmq.common.MQVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.help.FAQUrl;
@@ -60,6 +63,8 @@ import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
public class SendMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor {
private List<ConsumeMessageHook> consumeMessageHookList;
@@ -96,16 +101,54 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
if (requestHeader == null) {
return CompletableFuture.completedFuture(null);
}
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader, true);
RemotingCommand rewriteResult = this.brokerController.getTopicQueueMappingManager().rewriteRequestForStaticTopic(requestHeader, mappingContext);
if (rewriteResult != null) {
return CompletableFuture.completedFuture(rewriteResult);
}
mqtraceContext = buildMsgContext(ctx, requestHeader);
this.executeSendMessageHookBefore(ctx, request, mqtraceContext);
if (requestHeader.isBatch()) {
return this.asyncSendBatchMessage(ctx, request, mqtraceContext, requestHeader);
return this.asyncSendBatchMessage(ctx, request, mqtraceContext, requestHeader, mappingContext);
} else {
return this.asyncSendMessage(ctx, request, mqtraceContext, requestHeader);
return this.asyncSendMessage(ctx, request, mqtraceContext, requestHeader, mappingContext);
}
}
}
/**
* If the response is not null, it meets some errors
* @return
*/
private RemotingCommand rewriteResponseForStaticTopic(SendMessageResponseHeader responseHeader, TopicQueueMappingContext mappingContext) {
try {
if (mappingContext.getMappingDetail() == null) {
return null;
}
TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();
if (mappingItem == null) {
return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));
}
//no need to care the broker name
long staticLogicOffset = mappingItem.computeStaticQueueOffsetLoosely(responseHeader.getQueueOffset());
if (staticLogicOffset < 0) {
//if the logic offset is -1, just let it go
//maybe we need a dynamic config
//return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d convert offset error in current broker %s", mappingContext.getTopic(), mappingContext.getGlobalId(), mappingDetail.getBname()));
}
responseHeader.setQueueId(mappingContext.getGlobalId());
responseHeader.setQueueOffset(staticLogicOffset);
} catch (Throwable t) {
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
return null;
}
@Override
public boolean rejectRequest() {
return this.brokerController.getMessageStore().isOSPageCacheBusy() ||
@@ -171,6 +214,14 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
return CompletableFuture.completedFuture(response);
}
if (requestHeader.getOriginTopic() != null
&& !msgExt.getTopic().equals(requestHeader.getOriginTopic())) {
//here just do some fence in case of some unexpected offset is income
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("look message by offset failed to check the topic name" + requestHeader.getOffset());
return CompletableFuture.completedFuture(response);
}
final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC);
if (null == retryTopic) {
MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic());
@@ -264,7 +315,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
private CompletableFuture<RemotingCommand> asyncSendMessage(ChannelHandlerContext ctx, RemotingCommand request,
SendMessageContext mqtraceContext,
SendMessageRequestHeader requestHeader) {
SendMessageRequestHeader requestHeader,
TopicQueueMappingContext mappingContext) {
final RemotingCommand response = preSend(ctx, request, requestHeader);
final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
@@ -310,12 +362,6 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
}
LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(msgInner.getTopic(), msgInner.getQueueId(), response);
CompletableFuture<RemotingCommand> future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response);
if (future != null) {
return future;
}
CompletableFuture<PutMessageResult> putMessageResult = null;
String transFlag = origProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED);
if (transFlag != null && Boolean.parseBoolean(transFlag)) {
@@ -324,15 +370,13 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
response.setRemark(
"the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1()
+ "] sending transaction message is forbidden");
logicalQueueContext.hookAfterPut(null);
return CompletableFuture.completedFuture(response);
}
putMessageResult = this.brokerController.getTransactionalMessageService().asyncPrepareMessage(msgInner);
} else {
putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner);
}
logicalQueueContext.hookAfterPut(putMessageResult);
return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt);
return handlePutMessageResultFuture(putMessageResult, response, request, msgInner, responseHeader, mqtraceContext, ctx, queueIdInt, requestHeader, mappingContext);
}
private CompletableFuture<RemotingCommand> handlePutMessageResultFuture(CompletableFuture<PutMessageResult> putMessageResult,
@@ -342,9 +386,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
SendMessageResponseHeader responseHeader,
SendMessageContext sendMessageContext,
ChannelHandlerContext ctx,
int queueIdInt) {
int queueIdInt,
SendMessageRequestHeader requestHeader,
TopicQueueMappingContext mappingContext) {
return putMessageResult.thenApply((r) ->
handlePutMessageResult(r, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt)
handlePutMessageResult(r, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, requestHeader, mappingContext)
);
}
@@ -396,7 +442,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
private RemotingCommand sendMessage(final ChannelHandlerContext ctx,
final RemotingCommand request,
final SendMessageContext sendMessageContext,
final SendMessageRequestHeader requestHeader) throws RemotingCommandException {
final SendMessageRequestHeader requestHeader,
final TopicQueueMappingContext mappingContext) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class);
final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
@@ -465,14 +512,15 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner);
}
return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt);
return handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt, requestHeader, mappingContext);
}
private RemotingCommand handlePutMessageResult(PutMessageResult putMessageResult, RemotingCommand response,
RemotingCommand request, MessageExt msg,
SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx,
int queueIdInt) {
int queueIdInt, SendMessageRequestHeader requestHeader,
TopicQueueMappingContext mappingContext) {
if (putMessageResult == null) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark("store putMessage return null");
@@ -549,6 +597,11 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
responseHeader.setQueueId(queueIdInt);
responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset());
RemotingCommand rewriteResult = rewriteResponseForStaticTopic(responseHeader, mappingContext);
if (rewriteResult != null) {
return rewriteResult;
}
doResponse(ctx, request, response);
if (hasSendMessageHook()) {
@@ -582,7 +635,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
private CompletableFuture<RemotingCommand> asyncSendBatchMessage(ChannelHandlerContext ctx, RemotingCommand request,
SendMessageContext mqtraceContext,
SendMessageRequestHeader requestHeader) {
SendMessageRequestHeader requestHeader,
TopicQueueMappingContext mappingContext) {
final RemotingCommand response = preSend(ctx, request, requestHeader);
final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader();
@@ -623,17 +677,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement
String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName();
MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName);
LogicalQueueContext logicalQueueContext = super.buildLogicalQueueContext(messageExtBatch.getTopic(), messageExtBatch.getQueueId(), response);
CompletableFuture<RemotingCommand> future = logicalQueueContext.hookBeforePut(ctx, requestHeader, request, response);
if (future != null) {
return future;
}
CompletableFuture<PutMessageResult> putMessageResult = this.brokerController.getMessageStore().asyncPutMessages(messageExtBatch);
logicalQueueContext.hookAfterPut(putMessageResult);
return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt);
return handlePutMessageResultFuture(putMessageResult, response, request, messageExtBatch, responseHeader, mqtraceContext, ctx, queueIdInt, requestHeader, mappingContext);
}
broker/src/main/java/org/apache/rocketmq/broker/slave/SlaveSynchronize.java
+16 -7
View File
@@ -16,18 +16,19 @@
*/
package org.apache.rocketmq.broker.slave;
import java.io.IOException;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigAndMappingSerializeWrapper;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import java.io.IOException;
public class SlaveSynchronize {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private final BrokerController brokerController;
@@ -56,7 +57,7 @@ public class SlaveSynchronize {
String masterAddrBak = this.masterAddr;
if (masterAddrBak != null && !masterAddrBak.equals(brokerController.getBrokerAddr())) {
try {
TopicConfigSerializeWrapper topicWrapper =
TopicConfigAndMappingSerializeWrapper topicWrapper =
this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak);
if (!this.brokerController.getTopicConfigManager().getDataVersion()
.equals(topicWrapper.getDataVersion())) {
@@ -67,9 +68,17 @@ public class SlaveSynchronize {
this.brokerController.getTopicConfigManager().getTopicConfigTable()
.putAll(topicWrapper.getTopicConfigTable());
this.brokerController.getTopicConfigManager().persist();
log.info("Update slave topic config from master, {}", masterAddrBak);
}
if (topicWrapper.getTopicQueueMappingDetailMap() != null
&& !topicWrapper.getMappingDataVersion().equals(this.brokerController.getTopicQueueMappingManager().getDataVersion())) {
this.brokerController.getTopicQueueMappingManager().getDataVersion()
.assignNewOne(topicWrapper.getMappingDataVersion());
this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable().clear();
this.brokerController.getTopicQueueMappingManager().getTopicQueueMappingTable()
.putAll(topicWrapper.getTopicQueueMappingDetailMap());
this.brokerController.getTopicQueueMappingManager().persist();
}
log.info("Update slave topic config from master, {}", masterAddrBak);
} catch (Exception e) {
log.error("SyncTopicConfig Exception, {}", masterAddrBak, e);
}
broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java
+2 -113
View File
@@ -17,20 +17,16 @@
package org.apache.rocketmq.broker.topic;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.MixAll;
@@ -39,15 +35,10 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.sysflag.TopicSysFlag;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.srvutil.ConcurrentHashMapUtil;
import org.apache.rocketmq.store.CleanFilesHook;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.MessageStore;
public class TopicConfigManager extends ConfigManager {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -61,19 +52,8 @@ public class TopicConfigManager extends ConfigManager {
private final DataVersion dataVersion = new DataVersion();
private transient BrokerController brokerController;
private final ConcurrentMap<String, LogicalQueuesInfoInBroker> logicalQueuesInfoTable = new ConcurrentHashMap<>();
private final CleanFilesHook logicalQueueCleanHook = new CleanFilesHook() {
@Override public void execute(DefaultMessageStore defaultMessageStore, long deleteCount) {
if (deleteCount == 0) {
return;
}
TopicConfigManager.this.logicalQueueClean();
}
@Override public String getName() {
return TopicConfigManager.class.getSimpleName() + ".logicalQueueCleanHook";
}
};
public TopicConfigManager() {
}
public TopicConfigManager(BrokerController brokerController) {
this.brokerController = brokerController;
@@ -441,8 +421,6 @@ public class TopicConfigManager extends ConfigManager {
public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() {
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable);
String brokerName = this.brokerController.getBrokerConfig().getBrokerName();
topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue(), data -> Objects.equals(data.getBrokerName(), brokerName)))));
topicConfigSerializeWrapper.setDataVersion(this.dataVersion);
return topicConfigSerializeWrapper;
}
@@ -474,7 +452,6 @@ public class TopicConfigManager extends ConfigManager {
public String encode(final boolean prettyFormat) {
TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable);
topicConfigSerializeWrapper.setLogicalQueuesInfoMap(this.logicalQueuesInfoTable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> new LogicalQueuesInfoInBroker(e.getValue()))));
topicConfigSerializeWrapper.setDataVersion(this.dataVersion);
return topicConfigSerializeWrapper.toJson(prettyFormat);
}
@@ -494,92 +471,4 @@ public class TopicConfigManager extends ConfigManager {
public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {
return topicConfigTable;
}
public LogicalQueuesInfoInBroker selectLogicalQueuesInfo(String topicName) {
return this.logicalQueuesInfoTable.get(topicName);
}
public LogicalQueuesInfoInBroker getOrCreateLogicalQueuesInfo(String topicName) {
return ConcurrentHashMapUtil.computeIfAbsent(this.logicalQueuesInfoTable, topicName, ignored -> new LogicalQueuesInfoInBroker());
}
public boolean replaceTopicConfig(String topic, TopicConfig oldTopicConfig, TopicConfig newTopicConfig) {
boolean ok = this.topicConfigTable.replace(topic, oldTopicConfig, newTopicConfig);
if (ok) {
this.dataVersion.nextVersion();
persist(topic, newTopicConfig);
}
return ok;
}
public CleanFilesHook getLogicalQueueCleanHook() {
return logicalQueueCleanHook;
}
void logicalQueueClean() {
String brokerName = this.brokerController.getBrokerConfig().getBrokerName();
MessageStore messageStore = this.brokerController.getMessageStore();
for (Entry<String, LogicalQueuesInfoInBroker> entry : this.logicalQueuesInfoTable.entrySet()) {
String topic = entry.getKey();
LogicalQueuesInfoInBroker logicalQueuesInfo = entry.getValue();
Lock readLock = logicalQueuesInfo.readLock();
Lock writeLock = logicalQueuesInfo.writeLock();
boolean changed = false;
readLock.lock();
try {
for (List<LogicalQueueRouteData> list : logicalQueuesInfo.values()) {
while (!list.isEmpty()) {
LogicalQueueRouteData logicalQueueRouteData = list.get(0);
String brokerBelongs;
if (brokerName.equals(logicalQueueRouteData.getBrokerName())) {
if (logicalQueueRouteData.isWritable()) {
break;
}
boolean canRemove = logicalQueueRouteData.isExpired() || logicalQueueRouteData.getMessagesCount() == 0;
if (!canRemove) {
// do not use getMinOffsetInQueue method, since it is using ConsumeQueue data, but not CommitLog, CQ data is not accurate after CommitLog cleaning.
long commitLogOffset = messageStore.getCommitLogOffsetInQueue(topic, logicalQueueRouteData.getQueueId(), logicalQueueRouteData.getOffsetMax() - 1);
canRemove = commitLogOffset == 0 || messageStore.getMinPhyOffset() > commitLogOffset;
}
if (!canRemove) {
break;
}
brokerBelongs = "self";
} else {
brokerBelongs = "other";
}
readLock.unlock();
writeLock.lock();
try {
list.remove(0);
} finally {
readLock.lock();
writeLock.unlock();
}
log.info("logicalQueueClean remove {} broker {}", brokerBelongs, logicalQueueRouteData);
changed = true;
}
}
if (changed) {
logicalQueuesInfo = new LogicalQueuesInfoInBroker(logicalQueuesInfo);
}
} finally {
readLock.unlock();
}
if (changed) {
this.dataVersion.nextVersion();
this.persist(topic, logicalQueuesInfo);
this.brokerController.registerIncrementBrokerData(this.selectTopicConfig(topic), this.dataVersion);
log.info("registerIncrementBrokerData because logicalQueueClean: {}", topic);
}
}
}
public void deleteQueueRouteData(String topic) {
if (this.logicalQueuesInfoTable.remove(topic) != null) {
log.info("delete queueRouteData config OK, topic: {}", topic);
this.dataVersion.nextVersion();
persist(topic, (LogicalQueuesInfoInBroker) null);
}
}
}
broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingCleanService.java
+332
View File
@@ -0,0 +1,332 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.topic;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.out.BrokerOuterAPI;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.admin.TopicOffset;
import org.apache.rocketmq.common.admin.TopicStatsTable;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.rpc.ClientMetadata;
import org.apache.rocketmq.common.rpc.RpcClient;
import org.apache.rocketmq.common.rpc.RpcRequest;
import org.apache.rocketmq.common.rpc.RpcResponse;
import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class TopicQueueMappingCleanService extends ServiceThread {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private TopicQueueMappingManager topicQueueMappingManager;
private BrokerOuterAPI brokerOuterAPI;
private RpcClient rpcClient;
private MessageStoreConfig messageStoreConfig;
private BrokerConfig brokerConfig;
public TopicQueueMappingCleanService(BrokerController brokerController) {
this.topicQueueMappingManager = brokerController.getTopicQueueMappingManager();
this.rpcClient = brokerController.getBrokerOuterAPI().getRpcClient();
this.messageStoreConfig = brokerController.getMessageStoreConfig();
this.brokerConfig = brokerController.getBrokerConfig();
this.brokerOuterAPI = brokerController.getBrokerOuterAPI();
}
@Override
public String getServiceName() {
return TopicQueueMappingCleanService.class.getSimpleName();
}
@Override
public void run() {
log.info("Start topic queue mapping clean service thread!");
while (!this.isStopped()) {
try {
this.waitForRunning(5L * 60 * 1000);
} catch (Throwable ignored) {
}
try {
cleanItemExpired();
} catch (Throwable t) {
log.error("topic queue mapping cleanItemExpired failed", t);
}
try {
cleanItemListMoreThanSecondGen();
} catch (Throwable t) {
log.error("topic queue mapping cleanItemListMoreThanSecondGen failed", t);
}
}
log.info("End topic queue mapping clean service thread!");
}
public void cleanItemExpired() {
String when = messageStoreConfig.getDeleteWhen();
if (!UtilAll.isItTimeToDo(when)) {
return;
}
boolean changed = false;
long start = System.currentTimeMillis();
try {
for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {
try {
if (isStopped()) {
break;
}
TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);
if (mappingDetail == null
|| mappingDetail.getHostedQueues().isEmpty()) {
continue;
}
if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {
log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail);
continue;
}
Set<String> brokers = new HashSet<>();
for (List<LogicQueueMappingItem> items: mappingDetail.getHostedQueues().values()) {
if (items.size() <= 1) {
continue;
}
if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {
continue;
}
LogicQueueMappingItem earlistItem = items.get(0);
brokers.add(earlistItem.getBname());
}
Map<String, TopicStatsTable> statsTable = new HashMap<>();
for (String broker: brokers) {
GetTopicStatsInfoRequestHeader header = new GetTopicStatsInfoRequestHeader();
header.setTopic(topic);
header.setBname(broker);
header.setLo(false);
try {
RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_STATS_INFO, header, null);
RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();
if (rpcResponse.getException() != null) {
throw rpcResponse.getException();
}
statsTable.put(broker, (TopicStatsTable) rpcResponse.getBody());
} catch (Throwable rt) {
log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt);
}
}
Map<Integer, List<LogicQueueMappingItem>> newHostedQueues = new HashMap<>();
boolean changedForTopic = false;
for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {
Integer qid = entry.getKey();
List<LogicQueueMappingItem> items = entry.getValue();
if (items.size() <= 1) {
continue;
}
if (!TopicQueueMappingUtils.checkIfLeader(items, mappingDetail)) {
continue;
}
LogicQueueMappingItem earlistItem = items.get(0);
TopicStatsTable topicStats = statsTable.get(earlistItem.getBname());
if (topicStats == null) {
continue;
}
TopicOffset topicOffset = topicStats.getOffsetTable().get(new MessageQueue(topic, earlistItem.getBname(), earlistItem.getQueueId()));
if (topicOffset == null) {
//this may should not happen
log.error("Get null topicOffset for {} {}",topic, earlistItem);
continue;
}
//ignore the maxOffset < 0, which may in case of some error
if (topicOffset.getMaxOffset() == topicOffset.getMinOffset()
|| topicOffset.getMaxOffset() == 0) {
List<LogicQueueMappingItem> newItems = new ArrayList<>(items);
boolean result = newItems.remove(earlistItem);
if (result) {
changedForTopic = true;
newHostedQueues.put(qid, newItems);
}
log.info("The logic queue item {} {} is removed {} because of {}", topic, earlistItem, result, topicOffset);
}
}
if (changedForTopic) {
TopicQueueMappingDetail newMappingDetail = new TopicQueueMappingDetail(mappingDetail.getTopic(), mappingDetail.getTotalQueues(), mappingDetail.getBname(), mappingDetail.getEpoch());
newMappingDetail.getHostedQueues().putAll(mappingDetail.getHostedQueues());
newMappingDetail.getHostedQueues().putAll(newHostedQueues);
this.topicQueueMappingManager.updateTopicQueueMapping(newMappingDetail, false, true, false);
changed = true;
}
} catch (Throwable tt) {
log.error("Try CleanItemExpired failed for {}", topic, tt);
} finally {
UtilAll.sleep(10);
}
}
} catch (Throwable t) {
log.error("Try cleanItemExpired failed", t);
} finally {
if (changed) {
this.topicQueueMappingManager.getDataVersion().nextVersion();
this.topicQueueMappingManager.persist();
log.info("CleanItemExpired changed");
}
log.info("cleanItemExpired cost {} ms", System.currentTimeMillis() - start);
}
}
public void cleanItemListMoreThanSecondGen() {
String when = messageStoreConfig.getDeleteWhen();
if (!UtilAll.isItTimeToDo(when)) {
return;
}
boolean changed = false;
long start = System.currentTimeMillis();
try {
ClientMetadata clientMetadata = new ClientMetadata();
for (String topic : this.topicQueueMappingManager.getTopicQueueMappingTable().keySet()) {
try {
if (isStopped()) {
break;
}
TopicQueueMappingDetail mappingDetail = this.topicQueueMappingManager.getTopicQueueMappingTable().get(topic);
if (mappingDetail == null
|| mappingDetail.getHostedQueues().isEmpty()) {
continue;
}
if (!mappingDetail.getBname().equals(brokerConfig.getBrokerName())) {
log.warn("The TopicQueueMappingDetail [{}] should not exist in this broker", mappingDetail);
continue;
}
Map<Integer, String> qid2CurrLeaderBroker = new HashMap<>();
for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {
Integer qId = entry.getKey();
List<LogicQueueMappingItem> items = entry.getValue();
if (items.isEmpty()) {
continue;
}
LogicQueueMappingItem leaderItem = items.get(items.size() - 1);
if (!leaderItem.getBname().equals(mappingDetail.getBname())) {
qid2CurrLeaderBroker.put(qId, leaderItem.getBname());
}
}
if (qid2CurrLeaderBroker.isEmpty()) {
continue;
}
//find the topic route
TopicRouteData topicRouteData = brokerOuterAPI.getTopicRouteInfoFromNameServer(topic, brokerConfig.getForwardTimeout());
clientMetadata.freshTopicRoute(topic, topicRouteData);
Map<Integer, String> qid2RealLeaderBroker = new HashMap<>();
//fine the real leader
for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {
qid2RealLeaderBroker.put(entry.getKey(), clientMetadata.getBrokerNameFromMessageQueue(new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(mappingDetail.getScope()), entry.getKey())));
}
//find the mapping detail of real leader
Map<String, TopicQueueMappingDetail> mappingDetailMap = new HashMap<>();
for (Map.Entry<Integer, String> entry : qid2RealLeaderBroker.entrySet()) {
if (entry.getValue().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {
continue;
}
String broker = entry.getValue();
GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader();
header.setTopic(topic);
header.setBname(broker);
header.setLo(true);
try {
RpcRequest rpcRequest = new RpcRequest(RequestCode.GET_TOPIC_CONFIG, header, null);
RpcResponse rpcResponse = rpcClient.invoke(rpcRequest, brokerConfig.getForwardTimeout()).get();
if (rpcResponse.getException() != null) {
throw rpcResponse.getException();
}
TopicQueueMappingDetail mappingDetailRemote = ((TopicConfigAndQueueMapping) rpcResponse.getBody()).getMappingDetail();
if (broker.equals(mappingDetailRemote.getBname())) {
mappingDetailMap.put(broker, mappingDetailRemote);
}
} catch (Throwable rt) {
log.error("Get remote topic {} state info failed from broker {}", topic, broker, rt);
}
}
//check all the info
Set<Integer> ids2delete = new HashSet<>();
for (Map.Entry<Integer, String> entry : qid2CurrLeaderBroker.entrySet()) {
Integer qId = entry.getKey();
String currLeaderBroker = entry.getValue();
String realLeaderBroker = qid2RealLeaderBroker.get(qId);
TopicQueueMappingDetail remoteMappingDetail = mappingDetailMap.get(realLeaderBroker);
if (remoteMappingDetail == null
|| remoteMappingDetail.getTotalQueues() != mappingDetail.getTotalQueues()
|| remoteMappingDetail.getEpoch() != mappingDetail.getEpoch()) {
continue;
}
List<LogicQueueMappingItem> items = remoteMappingDetail.getHostedQueues().get(qId);
if (items.isEmpty()) {
continue;
}
LogicQueueMappingItem leaderItem = items.get(items.size() - 1);
if (!realLeaderBroker.equals(leaderItem.getBname())) {
continue;
}
//all the check is ok
if (!realLeaderBroker.equals(currLeaderBroker)) {
ids2delete.add(qId);
}
}
for (Integer qid : ids2delete) {
List<LogicQueueMappingItem> items = mappingDetail.getHostedQueues().remove(qid);
changed = true;
if (items != null) {
log.info("Remove the ItemListMoreThanSecondGen topic {} qid {} items {}", topic, qid, items);
}
}
} catch (Throwable tt) {
log.error("Try cleanItemListMoreThanSecondGen failed for topic {}", topic, tt);
} finally {
UtilAll.sleep(10);
}
}
} catch (Throwable t) {
log.error("Try cleanItemListMoreThanSecondGen failed", t);
} finally {
if (changed) {
this.topicQueueMappingManager.getDataVersion().nextVersion();
this.topicQueueMappingManager.persist();
}
log.info("Try cleanItemListMoreThanSecondGen cost {} ms", System.currentTimeMillis() - start);
}
}
}
broker/src/main/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManager.java
+259
View File
@@ -0,0 +1,259 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.topic;
import com.alibaba.fastjson.JSON;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.BrokerPathConfigHelper;
import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.TopicQueueMappingSerializeWrapper;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.common.rpc.TopicRequestHeader;
import org.apache.rocketmq.common.statictopic.LogicQueueMappingItem;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingContext;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
public class TopicQueueMappingManager extends ConfigManager {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
private static final long LOCK_TIMEOUT_MILLIS = 3000;
private transient final Lock lock = new ReentrantLock();
//this data version should be equal to the TopicConfigManager
private final DataVersion dataVersion = new DataVersion();
private transient BrokerController brokerController;
private final ConcurrentMap<String, TopicQueueMappingDetail> topicQueueMappingTable = new ConcurrentHashMap<>();
public TopicQueueMappingManager(BrokerController brokerController) {
this.brokerController = brokerController;
}
public void updateTopicQueueMapping(TopicQueueMappingDetail newDetail, boolean force, boolean isClean, boolean flush) throws Exception {
boolean locked = false;
boolean updated = false;
TopicQueueMappingDetail oldDetail = null;
try {
if (lock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
locked = true;
} else {
return;
}
if (newDetail == null) {
return;
}
assert newDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());
newDetail.getHostedQueues().forEach((queueId, items) -> {
TopicQueueMappingUtils.checkLogicQueueMappingItemOffset(items);
});
oldDetail = topicQueueMappingTable.get(newDetail.getTopic());
if (oldDetail == null) {
topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
updated = true;
return;
}
if (force) {
//bakeup the old items
oldDetail.getHostedQueues().forEach((queueId, items) -> {
newDetail.getHostedQueues().putIfAbsent(queueId, items);
});
topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
updated = true;
return;
}
//do more check
if (newDetail.getEpoch() < oldDetail.getEpoch()) {
throw new RuntimeException(String.format("Can't accept data with small epoch %d < %d", newDetail.getEpoch(), oldDetail.getEpoch()));
}
if (!newDetail.getScope().equals(oldDetail.getScope())) {
throw new RuntimeException(String.format("Can't accept data with unmatched scope %s != %s", newDetail.getScope(), oldDetail.getScope()));
}
boolean epochEqual = newDetail.getEpoch() == oldDetail.getEpoch();
for (Integer globalId : oldDetail.getHostedQueues().keySet()) {
List<LogicQueueMappingItem> oldItems = oldDetail.getHostedQueues().get(globalId);
List<LogicQueueMappingItem> newItems = newDetail.getHostedQueues().get(globalId);
if (newItems == null) {
if (epochEqual) {
throw new RuntimeException("Cannot accept equal epoch with null data");
} else {
newDetail.getHostedQueues().put(globalId, oldItems);
}
} else {
TopicQueueMappingUtils.makeSureLogicQueueMappingItemImmutable(oldItems, newItems, epochEqual, isClean);
}
}
topicQueueMappingTable.put(newDetail.getTopic(), newDetail);
updated = true;
} finally {
if (locked) {
this.lock.unlock();
}
if (updated && flush) {
this.dataVersion.nextVersion();
this.persist();
log.info("Update topic queue mapping from [{}] to [{}], force {}", oldDetail, newDetail, force);
}
}
}
public void delete(final String topic) {
TopicQueueMappingDetail old = this.topicQueueMappingTable.remove(topic);
if (old != null) {
log.info("delete topic queue mapping OK, topic queue mapping: {}", old);
this.dataVersion.nextVersion();
this.persist();
} else {
log.warn("delete topic queue mapping failed, topic: {} not exists", topic);
}
}
public TopicQueueMappingDetail getTopicQueueMapping(String topic) {
return topicQueueMappingTable.get(topic);
}
@Override
public String encode(boolean pretty) {
TopicQueueMappingSerializeWrapper wrapper = new TopicQueueMappingSerializeWrapper();
wrapper.setTopicQueueMappingInfoMap(topicQueueMappingTable);
wrapper.setDataVersion(this.dataVersion);
return JSON.toJSONString(wrapper, pretty);
}
@Override
public String encode() {
return encode(false);
}
@Override
public String configFilePath() {
return BrokerPathConfigHelper.getTopicQueueMappingPath(this.brokerController.getMessageStoreConfig()
.getStorePathRootDir());
}
@Override
public void decode(String jsonString) {
if (jsonString != null) {
TopicQueueMappingSerializeWrapper wrapper = TopicQueueMappingSerializeWrapper.fromJson(jsonString, TopicQueueMappingSerializeWrapper.class);
if (wrapper != null) {
this.topicQueueMappingTable.putAll(wrapper.getTopicQueueMappingInfoMap());
this.dataVersion.assignNewOne(wrapper.getDataVersion());
}
}
}
public ConcurrentMap<String, TopicQueueMappingDetail> getTopicQueueMappingTable() {
return topicQueueMappingTable;
}
public DataVersion getDataVersion() {
return dataVersion;
}
public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader) {
return buildTopicQueueMappingContext(requestHeader, false);
}
//Do not return a null context
public TopicQueueMappingContext buildTopicQueueMappingContext(TopicRequestHeader requestHeader, boolean selectOneWhenMiss) {
//should disable logic queue explicitly, otherwise the old client may cause dirty data to newly created static topic
if (requestHeader.getLo() != null
&& Boolean.FALSE.equals(requestHeader.getLo())) {
return new TopicQueueMappingContext(requestHeader.getTopic(), null, null, null, null);
}
String topic = requestHeader.getTopic();
Integer globalId = null;
if (requestHeader instanceof TopicQueueRequestHeader) {
globalId = ((TopicQueueRequestHeader) requestHeader).getQueueId();
}
TopicQueueMappingDetail mappingDetail = getTopicQueueMapping(topic);
if (mappingDetail == null) {
//it is not static topic
return new TopicQueueMappingContext(topic, null, null, null, null);
}
assert mappingDetail.getBname().equals(this.brokerController.getBrokerConfig().getBrokerName());
if (globalId == null) {
return new TopicQueueMappingContext(topic, null, mappingDetail, null, null);
}
//If not find mappingItem, it encounters some errors
if (globalId < 0 && !selectOneWhenMiss) {
return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null);
}
if (globalId < 0) {
try {
if (!mappingDetail.getHostedQueues().isEmpty()) {
//do not check
globalId = mappingDetail.getHostedQueues().keySet().iterator().next();
}
} catch (Throwable ignored) {
}
}
if (globalId < 0) {
return new TopicQueueMappingContext(topic, globalId, mappingDetail, null, null);
}
List<LogicQueueMappingItem> mappingItemList = TopicQueueMappingDetail.getMappingInfo(mappingDetail, globalId);
LogicQueueMappingItem leaderItem = null;
if (mappingItemList != null
&& mappingItemList.size() > 0) {
leaderItem = mappingItemList.get(mappingItemList.size() - 1);
}
return new TopicQueueMappingContext(topic, globalId, mappingDetail, mappingItemList, leaderItem);
}
public RemotingCommand rewriteRequestForStaticTopic(TopicQueueRequestHeader requestHeader, TopicQueueMappingContext mappingContext) {
try {
if (mappingContext.getMappingDetail() == null) {
return null;
}
TopicQueueMappingDetail mappingDetail = mappingContext.getMappingDetail();
if (!mappingContext.isLeader()) {
return buildErrorResponse(ResponseCode.NOT_LEADER_FOR_QUEUE, String.format("%s-%d does not exit in request process of current broker %s", requestHeader.getTopic(), requestHeader.getQueueId(), mappingDetail.getBname()));
}
LogicQueueMappingItem mappingItem = mappingContext.getLeaderItem();
requestHeader.setQueueId(mappingItem.getQueueId());
return null;
} catch (Throwable t) {
return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage());
}
}
}
broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java
+1 -1
View File
@@ -84,7 +84,7 @@ public class BrokerOuterAPITest {
private BrokerOuterAPI brokerOuterAPI;
public void init() throws Exception {
brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), null);
brokerOuterAPI = new BrokerOuterAPI(new NettyClientConfig(), brokerController);
Field field = BrokerOuterAPI.class.getDeclaredField("remotingClient");
field.setAccessible(true);
field.set(brokerOuterAPI, nettyRemotingClient);
broker/src/test/java/org/apache/rocketmq/broker/BrokerStartupTest.java
+17 -3
View File
@@ -20,6 +20,7 @@ package org.apache.rocketmq.broker;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Properties;
import org.apache.rocketmq.common.MixAll;
import org.junit.Assert;
import org.junit.Test;
@@ -34,8 +35,21 @@ public class BrokerStartupTest {
Class<BrokerStartup> clazz = BrokerStartup.class;
Method method = clazz.getDeclaredMethod("properties2SystemEnv", Properties.class);
method.setAccessible(true);
System.setProperty("rocketmq.namesrv.domain", "value");
method.invoke(null, properties);
Assert.assertEquals("value", System.getProperty("rocketmq.namesrv.domain"));
{
properties.put("rmqAddressServerDomain", "value1");
properties.put("rmqAddressServerSubGroup", "value2");
method.invoke(null, properties);
Assert.assertEquals("value1", System.getProperty("rocketmq.namesrv.domain"));
Assert.assertEquals("value2", System.getProperty("rocketmq.namesrv.domain.subgroup"));
}
{
properties.put("rmqAddressServerDomain", MixAll.WS_DOMAIN_NAME);
properties.put("rmqAddressServerSubGroup", MixAll.WS_DOMAIN_SUBGROUP);
method.invoke(null, properties);
Assert.assertEquals(MixAll.WS_DOMAIN_NAME, System.getProperty("rocketmq.namesrv.domain"));
Assert.assertEquals(MixAll.WS_DOMAIN_SUBGROUP, System.getProperty("rocketmq.namesrv.domain.subgroup"));
}
}
}
broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java
+9 -289
View File
@@ -16,22 +16,10 @@
*/
package org.apache.rocketmq.broker.processor;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import com.google.common.collect.Sets;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.LongAdder;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.topic.TopicConfigManager;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.MixAll;
@@ -42,28 +30,17 @@ import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody;
import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody;
import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.GetTopicConfigRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader;
import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.store.AppendMessageResult;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.MappedFile;
@@ -73,7 +50,6 @@ import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.assertj.core.util.Lists;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -81,13 +57,16 @@ import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.LongAdder;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -122,8 +101,7 @@ public class AdminBrokerProcessorTest {
public void init() throws Exception {
brokerController.setMessageStore(messageStore);
doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor();
when(sendMessageProcessor.getInFlyWritingCounterMap()).thenReturn(inFlyWritingCouterMap);
//doReturn(sendMessageProcessor).when(brokerController).getSendMessageProcessor();
adminBrokerProcessor = new AdminBrokerProcessor(brokerController);
@@ -225,271 +203,13 @@ public class AdminBrokerProcessorTest {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, requestHeader);
request.makeCustomHeaderToNet();
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
assertThat(response.getCode()).isEqualTo(ResponseCode.TOPIC_NOT_EXIST);
assertThat(response.getRemark()).contains("No topic in this broker.");
}
}
@Test
public void testUpdateTopicLogicalQueueMapping() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null);
UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody();
requestBody.setTopic(topic);
requestBody.setQueueId(queueId);
requestBody.setLogicalQueueIdx(logicalQueueIndex);
request.setBody(requestBody.encode());
RemotingCommand response;
response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr()));
// delete
requestBody.setLogicalQueueIdx(-1);
request.setBody(requestBody.encode());
response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
assertThat(topicConfigManager.getOrCreateLogicalQueuesInfo(topic).get(logicalQueueIndex)).isEmpty();
verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId));
}
@Test
public void testDeleteTopicLogicalQueueMapping() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())));
DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader();
requestHeader.setTopic(topic);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader);
request.makeCustomHeaderToNet();
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
assertThat(response.getRemark()).isEqualTo("still 1 message queues");
logicalQueuesInfo.remove(logicalQueueIndex);
response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull();
}
@Test
public void testQueryTopicLogicalQueueMapping() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())));
QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader();
requestHeader.setTopic(topic);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader);
request.makeCustomHeaderToNet();
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
Map<Integer, List<LogicalQueueRouteData>> m = JSON.parseObject(response.getBody(), new TypeReference<Map<Integer, List<LogicalQueueRouteData>>>() {
}.getType());
assertThat(m.get(logicalQueueIndex)).isEqualTo(logicalQueuesInfo.get(logicalQueueIndex));
}
@Test
public void testSealTopicLogicalQueue() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId);
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())));
when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L);
when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(0L);
when(messageStore.getMinPhyOffset()).thenReturn(1000L);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(0L))).thenReturn(2000L);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L);
MessageExt firstMsg = mock(MessageExt.class);
when(firstMsg.getStoreTimestamp()).thenReturn(200L);
when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg);
MessageExt lastMsg = mock(MessageExt.class);
when(lastMsg.getStoreTimestamp()).thenReturn(300L);
when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null);
SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody();
requestBody.setTopic(topic);
requestBody.setQueueId(queueId);
requestBody.setLogicalQueueIndex(logicalQueueIndex);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.ReadOnly, 0, 100, 200, 300, brokerController.getBrokerAddr());
assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData);
assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData);
// expired
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())));
when(messageStore.getMinPhyOffset()).thenReturn(10000L);
response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr());
assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData);
assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData);
// expired and empty
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Normal, 0L, -1, -1, -1, brokerController.getBrokerAddr())));
when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(100L);
response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 0L, mq, MessageQueueRouteState.Expired, 0, 100, 0, 0, brokerController.getBrokerAddr());
assertThat(logicalQueuesInfo.get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData);
assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData);
}
@Test
public void testReuseTopicLogicalQueue() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId);
LogicalQueueRouteData logicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Expired, 100L, 200L, 300L, 400L, brokerController.getBrokerAddr());
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(logicalQueueRouteData));
LogicalQueueRouteData wantData0 = new LogicalQueueRouteData(logicalQueueRouteData);
when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(600L);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null);
ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody();
requestBody.setTopic(topic);
requestBody.setQueueId(queueId);
requestBody.setLogicalQueueIndex(logicalQueueIndex);
requestBody.setMessageQueueRouteState(MessageQueueRouteState.WriteOnly);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
LogicalQueueRouteData wantData1 = new LogicalQueueRouteData(logicalQueueIndex, -1L, mq, MessageQueueRouteState.WriteOnly, 600L, -1, -1, -1, brokerController.getBrokerAddr());
assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantData1);
assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Arrays.asList(wantData0, wantData1));
verify(inFlyWritingCouterMap).remove(new TopicQueueId(topic, queueId));
}
@Test
public void testCreateMessageQueueForLogicalQueue() throws Exception {
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
TopicConfig topicConfig = topicConfigManager.selectTopicConfig(topic);
topicConfig.setWriteQueueNums(0);
topicConfig.setReadQueueNums(0);
int queueId = 0;
assertThat(topicConfigManager.selectLogicalQueuesInfo(topic)).isNull();
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null);
CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody();
requestBody.setTopic(topic);
requestBody.setLogicalQueueIndex(logicalQueueIndex);
requestBody.setMessageQueueStatus(MessageQueueRouteState.WriteOnly);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS);
LogicalQueueRouteData wantLogicalQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, -1L, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId), MessageQueueRouteState.WriteOnly, 0L, -1, -1, -1, brokerController.getBrokerAddr());
assertThat((LogicalQueueRouteData) JSON.parseObject(response.getBody(), LogicalQueueRouteData.class)).isEqualTo(wantLogicalQueueRouteData);
assertThat(topicConfigManager.selectLogicalQueuesInfo(topic).get(logicalQueueIndex).get(0)).isEqualTo(wantLogicalQueueRouteData);
}
@Test
public void testMigrateTopicLogicalQueuePrepare() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId);
LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 10L, -1L, -1L, -1L, brokerController.getBrokerAddr());
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData)));
when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId), anyBoolean())).thenReturn(100L);
when(messageStore.getMinOffsetInQueue(eq(topic), eq(queueId))).thenReturn(10L);
when(messageStore.getMinPhyOffset()).thenReturn(1000L);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(10L))).thenReturn(2000L);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(99L))).thenReturn(3000L);
MessageExt firstMsg = mock(MessageExt.class);
when(firstMsg.getStoreTimestamp()).thenReturn(200L);
when(messageStore.lookMessageByOffset(eq(2000L))).thenReturn(firstMsg);
MessageExt lastMsg = mock(MessageExt.class);
when(lastMsg.getStoreTimestamp()).thenReturn(300L);
when(messageStore.lookMessageByOffset(eq(3000L))).thenReturn(lastMsg);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, null);
MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody();
requestBody.setFromQueueRouteData(fromQueueRouteData);
LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData();
toQueueRouteData.setMessageQueue(new MessageQueue(topic, "toBroker", 1));
requestBody.setToQueueRouteData(toQueueRouteData);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS);
fromQueueRouteData.setState(MessageQueueRouteState.ReadOnly);
fromQueueRouteData.setOffsetMax(100L);
fromQueueRouteData.setFirstMsgTimeMillis(200L);
fromQueueRouteData.setLastMsgTimeMillis(300L);
toQueueRouteData.setLogicalQueueDelta(590L);
MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class);
assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData);
assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData);
assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData));
}
@Test
public void testMigrateTopicLogicalQueueCommit() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
MessageQueue mq = new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), queueId);
LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData();
fromQueueRouteData.setMessageQueue(new MessageQueue(topic, "fromBroker", 0));
LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, mq, MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr());
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(toQueueRouteData)));
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, null);
MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody();
requestBody.setFromQueueRouteData(fromQueueRouteData);
requestBody.setToQueueRouteData(toQueueRouteData);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS);
MigrateLogicalQueueBody responseBody = RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class);
assertThat(responseBody.getFromQueueRouteData()).isEqualTo(fromQueueRouteData);
assertThat(responseBody.getToQueueRouteData()).isEqualTo(toQueueRouteData);
assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(toQueueRouteData));
}
@Test
public void testMigrateTopicLogicalQueueNotify() throws Exception {
int queueId = 0;
int logicalQueueIndex = 0;
TopicConfigManager topicConfigManager = brokerController.getTopicConfigManager();
LogicalQueuesInfoInBroker logicalQueuesInfo = topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
LogicalQueueRouteData fromQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 100L, new MessageQueue(topic, "fromBroker", queueId), MessageQueueRouteState.ReadOnly, 10L, 410L, 200L, 300L, brokerController.getBrokerAddr());
LogicalQueueRouteData toQueueRouteData = new LogicalQueueRouteData(logicalQueueIndex, 500L, new MessageQueue(topic, "toBroker", queueId), MessageQueueRouteState.Normal, 500L, -1L, -1L, -1L, brokerController.getBrokerAddr());
logicalQueuesInfo.put(logicalQueueIndex, Lists.newArrayList(new LogicalQueueRouteData(fromQueueRouteData)));
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, null);
MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody();
requestBody.setFromQueueRouteData(fromQueueRouteData);
requestBody.setToQueueRouteData(toQueueRouteData);
request.setBody(requestBody.encode());
RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request);
assertThat(response.getCode()).withFailMessage("remark: %s", response.getRemark()).isEqualTo(ResponseCode.SUCCESS);
assertThat(logicalQueuesInfo.get(logicalQueueIndex)).isEqualTo(Lists.newArrayList(fromQueueRouteData, toQueueRouteData));
}
private RemotingCommand buildCreateTopicRequest(String topic) {
CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();
broker/src/test/java/org/apache/rocketmq/broker/processor/PullMessageProcessorTest.java
+6 -105
View File
@@ -18,33 +18,21 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageContext;
import org.apache.rocketmq.broker.mqtrace.ConsumeMessageHook;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.netty.NettyServerConfig;
@@ -53,7 +41,6 @@ import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.assertj.core.util.Lists;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -61,15 +48,17 @@ import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import static java.util.Optional.ofNullable;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.intThat;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -205,94 +194,6 @@ public class PullMessageProcessorTest {
assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_OFFSET_MOVED);
}
@Test
public void testProcessRequest_LogicalQueue() throws Exception {
String brokerName = brokerController.getBrokerConfig().getBrokerName();
int queueId = 1;
GetMessageResult getMessageResult = createGetMessageResult();
when(messageStore.getMessage(anyString(), eq(topic), eq(queueId), eq(456L), anyInt(), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult);
when(messageStore.getMaxOffsetInQueue(eq(topic), eq(queueId))).thenReturn(2000L);
when(messageStore.getMinPhyOffset()).thenReturn(0L);
LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic);
LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerName, queueId), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr());
logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1));
logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1);
// normal
{
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
assertThat(response).isNotNull();
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
}
// write only
queueRouteData1.setState(MessageQueueRouteState.WriteOnly);
{
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
assertThat(response).isNotNull();
assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND);
}
// no message and redirect
queueRouteData1.setState(MessageQueueRouteState.ReadOnly);
queueRouteData1.setOffsetMax(460);
queueRouteData1.setFirstMsgTimeMillis(100);
queueRouteData1.setLastMsgTimeMillis(200);
LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(0, 460, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr());
logicalQueuesInfo.get(0).add(queueRouteData2);
getMessageResult.setStatus(GetMessageStatus.OFFSET_FOUND_NULL);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId), eq(460L - 1L))).thenReturn(1000L);
{
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
assertThat(response).isNotNull();
assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_NOT_FOUND);
assertThat(response.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT);
}
// same message queue has two routes
queueRouteData2.setState(MessageQueueRouteState.ReadOnly);
queueRouteData2.setOffsetMax(50);
queueRouteData2.setFirstMsgTimeMillis(300);
queueRouteData2.setLastMsgTimeMillis(400);
LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(0, 510, new MessageQueue(topic, queueRouteData2.getBrokerName(), queueId), MessageQueueRouteState.Normal, 460, -1, -1, -1, queueRouteData1.getBrokerAddr());
logicalQueuesInfo.get(0).add(queueRouteData3);
logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData3.getQueueId(), queueRouteData3);
{
GetMessageResult getMessageResult2 = createGetMessageResult();
getMessageResult2.setStatus(GetMessageStatus.FOUND);
getMessageResult2.setNextBeginOffset(460);
when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), eq(4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2);
}
{
GetMessageResult getMessageResult2 = createGetMessageResult();
getMessageResult2.setStatus(GetMessageStatus.FOUND);
getMessageResult2.setNextBeginOffset(470);
lenient().when(messageStore.getMessage(anyString(), eq(queueRouteData1.getTopic()), eq(queueRouteData1.getQueueId()), eq(456L), intThat(i -> i > 4), any(ExpressionMessageFilter.class))).thenReturn(getMessageResult2);
}
{
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
assertThat(response).isNotNull();
assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS);
assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).doesNotContainKey(MessageConst.PROPERTY_REDIRECT);
PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader();
assertThat(header.getNextBeginOffset()).isEqualTo(460);
}
{
when(messageStore.getMinPhyOffset()).thenReturn(100000L);
final RemotingCommand request = createPullMsgCommand(RequestCode.PULL_MESSAGE);
RemotingCommand response = pullMessageProcessor.processRequest(handlerContext, request);
assertThat(response).isNotNull();
assertThat(response.getCode()).isEqualTo(ResponseCode.PULL_RETRY_IMMEDIATELY);
assertThat(ofNullable(response.getExtFields()).orElse(new HashMap<>())).containsKey(MessageConst.PROPERTY_REDIRECT);
PullMessageResponseHeader header = (PullMessageResponseHeader) response.readCustomHeader();
assertThat(header.getNextBeginOffset()).isEqualTo(460);
}
}
private RemotingCommand createPullMsgCommand(int requestCode) {
PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
requestHeader.setCommitOffset(123L);
broker/src/test/java/org/apache/rocketmq/broker/processor/SendMessageProcessorTest.java
+6 -67
View File
@@ -18,14 +18,7 @@ package org.apache.rocketmq.broker.processor;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
import org.apache.rocketmq.broker.transaction.TransactionalMessageService;
@@ -35,13 +28,10 @@ import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -55,7 +45,6 @@ import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.assertj.core.util.Lists;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -65,6 +54,12 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
@@ -229,62 +224,6 @@ public class SendMessageProcessorTest {
assertThat(response[0].getCode()).isEqualTo(ResponseCode.SUCCESS);
}
@Test
public void testProcessRequest_LogicalQueue() throws Exception {
when(messageStore.asyncPutMessage(any(MessageExtBrokerInner.class)))
.thenReturn(CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))));
LogicalQueuesInfoInBroker logicalQueuesInfo = brokerController.getTopicConfigManager().getOrCreateLogicalQueuesInfo(topic);
LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(0, 0, new MessageQueue(topic, brokerController.getBrokerConfig().getBrokerName(), 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr());
logicalQueuesInfo.put(0, Lists.newArrayList(queueRouteData1));
logicalQueuesInfo.updateQueueRouteDataByQueueId(queueRouteData1.getQueueId(), queueRouteData1);
SendMessageRequestHeader requestHeader = createSendMsgRequestHeader();
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader);
request.setBody(new byte[] {'a'});
request.makeCustomHeaderToNet();
// normal
RemotingCommand responseToReturn;
{
CompletableFuture<RemotingCommand> responseFuture = new CompletableFuture<>();
doAnswer(invocation -> {
responseFuture.complete(invocation.getArgument(0));
return null;
}).when(handlerContext).writeAndFlush(any(Object.class));
responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
if (responseToReturn == null) {
responseToReturn = responseFuture.get(3, TimeUnit.SECONDS);
}
}
assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS);
assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque());
// read only
queueRouteData1.setState(MessageQueueRouteState.ReadOnly);
responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
assertThat(responseToReturn.getRemark()).contains("not writable");
// read only and forward
logicalQueuesInfo.get(0).add(new LogicalQueueRouteData(0, 100, new MessageQueue(topic, "broker2", 1), MessageQueueRouteState.Normal, 0, -1, -1, -1, brokerController.getBrokerAddr()));
responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR);
assertThat(responseToReturn.getRemark()).contains("forward error");
// read only and redirect
requestHeader = (SendMessageRequestHeader) request.readCustomHeader();
requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG);
request.makeCustomHeaderToNet();
responseToReturn = sendMessageProcessor.processRequest(handlerContext, request);
assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.NO_PERMISSION);
assertThat(responseToReturn.getExtFields()).containsKey(MessageConst.PROPERTY_REDIRECT);
}
private RemotingCommand createSendTransactionMsgCommand(int requestCode) {
SendMessageRequestHeader header = createSendMsgRequestHeader();
int sysFlag = header.getSysFlag();
broker/src/test/java/org/apache/rocketmq/broker/topic/TopicConfigManagerTest.java
-138
View File
@@ -1,138 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.topic;
import com.google.common.collect.Lists;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.domain.LogicalQueuesInfoInBroker;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class TopicConfigManagerTest {
@Mock
private DefaultMessageStore messageStore;
@Mock
private BrokerController brokerController;
private TopicConfigManager topicConfigManager;
private static final String topic = "FooBar";
private static final String broker1Name = "broker1";
private static final String broker1Addr = "127.0.0.1:12345";
private static final int queueId1 = 1;
private static final String broker2Name = "broker2";
private static final String broker2Addr = "127.0.0.2:12345";
private static final int queueId2 = 2;
@Before
public void before() {
BrokerConfig brokerConfig = new BrokerConfig();
brokerConfig.setBrokerName(broker1Name);
when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
when(brokerController.getMessageStore()).thenReturn(messageStore);
MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir"));
when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
this.topicConfigManager = new TopicConfigManager(brokerController);
this.topicConfigManager.getTopicConfigTable().put(topic, new TopicConfig(topic));
}
@After
public void after() throws Exception {
if (topicConfigManager != null) {
Files.deleteIfExists(Paths.get(topicConfigManager.configFilePath()));
}
}
@Test
public void logicalQueueCleanTest() {
LogicalQueuesInfoInBroker info = this.topicConfigManager.getOrCreateLogicalQueuesInfo(topic);
topicConfigManager.logicalQueueClean();
assertThat(info).isEmpty();
final int logicalQueueIndex = 0;
LogicalQueueRouteData queueRouteData1 = new LogicalQueueRouteData(logicalQueueIndex, 0, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker1Addr);
List<LogicalQueueRouteData> l = Lists.newArrayList(new LogicalQueueRouteData(queueRouteData1));
info.put(logicalQueueIndex, l);
topicConfigManager.logicalQueueClean();
assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData1));
verify(messageStore, never()).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), anyLong());
verify(messageStore, never()).getMinPhyOffset();
verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.<TopicConfig>argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class));
LogicalQueueRouteData queueRouteData2 = new LogicalQueueRouteData(logicalQueueIndex, 100, new MessageQueue(topic, broker2Name, queueId2), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr);
l.add(new LogicalQueueRouteData(queueRouteData2));
queueRouteData1 = l.get(0);
queueRouteData1.setState(MessageQueueRouteState.ReadOnly);
queueRouteData1.setOffsetMax(100);
queueRouteData1.setFirstMsgTimeMillis(200);
queueRouteData1.setLastMsgTimeMillis(300);
queueRouteData1 = new LogicalQueueRouteData(queueRouteData1);
LogicalQueueRouteData queueRouteData3 = new LogicalQueueRouteData(logicalQueueIndex, 200, new MessageQueue(topic, broker1Name, queueId1), MessageQueueRouteState.Normal, 100, -1, -1, -1, broker1Addr);
l.add(new LogicalQueueRouteData(queueRouteData3));
queueRouteData2 = l.get(1);
queueRouteData2.setState(MessageQueueRouteState.ReadOnly);
queueRouteData2.setOffsetMax(100);
queueRouteData2.setFirstMsgTimeMillis(400);
queueRouteData2.setLastMsgTimeMillis(500);
queueRouteData2 = new LogicalQueueRouteData(queueRouteData2);
when(messageStore.getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1))).thenReturn(1000L);
when(messageStore.getMinPhyOffset()).thenReturn(0L);
topicConfigManager.logicalQueueClean();
assertThat(info.get(logicalQueueIndex)).isEqualTo(Arrays.asList(queueRouteData1, queueRouteData2, queueRouteData3));
verify(messageStore).getCommitLogOffsetInQueue(eq(topic), eq(queueId1), eq(queueRouteData1.getOffsetMax() - 1));
verify(messageStore).getMinPhyOffset();
verify(brokerController, never()).registerIncrementBrokerData(ArgumentMatchers.<TopicConfig>argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class));
when(messageStore.getMinPhyOffset()).thenReturn(2000L);
topicConfigManager.logicalQueueClean();
assertThat(info.get(logicalQueueIndex)).isEqualTo(Collections.singletonList(queueRouteData3));
verify(brokerController).registerIncrementBrokerData(ArgumentMatchers.<TopicConfig>argThat(arg -> topic.equals(arg.getTopicName())), any(DataVersion.class));
}
}
broker/src/test/java/org/apache/rocketmq/broker/topic/TopicQueueMappingManagerTest.java
+112
View File
@@ -0,0 +1,112 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.broker.topic;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.common.statictopic.TopicRemappingDetailWrapper;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class TopicQueueMappingManagerTest {
@Mock
private BrokerController brokerController;
private static final String broker1Name = "broker1";
@Before
public void before() {
BrokerConfig brokerConfig = new BrokerConfig();
brokerConfig.setBrokerName(broker1Name);
when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir"));
messageStoreConfig.setDeleteWhen("01;02;03;04;05;06;07;08;09;10;11;12;13;14;15;16;17;18;19;20;21;22;23;00");
when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig);
}
private void delete(TopicQueueMappingManager topicQueueMappingManager) throws Exception {
if (topicQueueMappingManager == null) {
return;
}
Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath()));
Files.deleteIfExists(Paths.get(topicQueueMappingManager.configFilePath() + ".bak"));
}
@Test
public void testEncodeDecode() throws Exception {
Map<String, TopicQueueMappingDetail> mappingDetailMap = new HashMap<>();
TopicQueueMappingManager topicQueueMappingManager = null;
Set<String> brokers = new HashSet<String>();
brokers.add(broker1Name);
{
for (int i = 0; i < 10; i++) {
String topic = UUID.randomUUID().toString();
int queueNum = 10;
TopicRemappingDetailWrapper topicRemappingDetailWrapper = TopicQueueMappingUtils.createTopicConfigMapping(topic, queueNum, brokers, new HashMap<>());
Assert.assertEquals(1, topicRemappingDetailWrapper.getBrokerConfigMap().size());
TopicQueueMappingDetail topicQueueMappingDetail = topicRemappingDetailWrapper.getBrokerConfigMap().values().iterator().next().getMappingDetail();
Assert.assertEquals(queueNum, topicQueueMappingDetail.getHostedQueues().size());
mappingDetailMap.put(topic, topicQueueMappingDetail);
}
}
{
topicQueueMappingManager = new TopicQueueMappingManager(brokerController);
Assert.assertTrue(topicQueueMappingManager.load());
Assert.assertEquals(0, topicQueueMappingManager.getTopicQueueMappingTable().size());
for (TopicQueueMappingDetail mappingDetail : mappingDetailMap.values()) {
for (int i = 0; i < 10; i++) {
topicQueueMappingManager.updateTopicQueueMapping(mappingDetail, false, false, true);
}
}
topicQueueMappingManager.persist();
}
{
topicQueueMappingManager = new TopicQueueMappingManager(brokerController);
Assert.assertTrue(topicQueueMappingManager.load());
Assert.assertEquals(mappingDetailMap.size(), topicQueueMappingManager.getTopicQueueMappingTable().size());
for (TopicQueueMappingDetail topicQueueMappingDetail: topicQueueMappingManager.getTopicQueueMappingTable().values()) {
Assert.assertEquals(topicQueueMappingDetail, mappingDetailMap.get(topicQueueMappingDetail.getTopic()));
}
}
delete(topicQueueMappingManager);
}
}
client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java
+1 -1
View File
@@ -691,7 +691,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume
public void sendMessageBack(MessageExt msg, int delayLevel)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
msg.setTopic(withNamespace(msg.getTopic()));
this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, null);
this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, (String) null);
}
/**
client/src/main/java/org/apache/rocketmq/client/consumer/PullResult.java
+1
View File
@@ -26,6 +26,7 @@ public class PullResult {
private final long maxOffset;
private List<MessageExt> msgFoundList;
public PullResult(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,
List<MessageExt> msgFoundList) {
super();
client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java
+20 -18
View File
@@ -16,6 +16,20 @@
*/
package org.apache.rocketmq.client.consumer.store;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.OffsetNotFoundException;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.exception.RemotingException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -23,18 +37,6 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.remoting.exception.RemotingException;
/**
* Remote storage implementation
@@ -94,7 +96,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore {
return brokerOffset;
}
// No offset in broker
catch (MQBrokerException e) {
catch (OffsetNotFoundException e) {
return -1;
}
//Other exceptions
@@ -108,7 +110,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore {
}
}
return -1;
return -3;
}
@Override
@@ -199,10 +201,10 @@ public class RemoteBrokerOffsetStore implements OffsetStore {
@Override
public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException,
MQBrokerException, InterruptedException, MQClientException {
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (findBrokerResult != null) {
@@ -226,11 +228,11 @@ public class RemoteBrokerOffsetStore implements OffsetStore {
private long fetchConsumeOffsetFromBroker(MessageQueue mq) throws RemotingException, MQBrokerException,
InterruptedException, MQClientException {
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName());
findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (findBrokerResult != null) {
common/src/main/java/org/apache/rocketmq/common/protocol/header/DeleteTopicLogicalQueueRequestHeader.java → client/src/main/java/org/apache/rocketmq/client/exception/OffsetNotFoundException.java
+7 -13
View File
@@ -14,24 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.header;
package org.apache.rocketmq.client.exception;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class OffsetNotFoundException extends MQBrokerException {
public class DeleteTopicLogicalQueueRequestHeader implements CommandCustomHeader {
@CFNotNull
private String topic;
@Override public void checkFields() throws RemotingCommandException {
public OffsetNotFoundException() {
}
public String getTopic() {
return topic;
public OffsetNotFoundException(int responseCode, String errorMessage) {
super(responseCode, errorMessage);
}
public void setTopic(String topic) {
this.topic = topic;
public OffsetNotFoundException(int responseCode, String errorMessage, String brokerAddr) {
super(responseCode, errorMessage, brokerAddr);
}
}
client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java
+52 -178
View File
@@ -16,41 +16,6 @@
*/
package org.apache.rocketmq.client.impl;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import org.apache.rocketmq.client.QueryResult;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.message.MessageClientIDSetter;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageId;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.NamespaceUtil;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.InvokeCallback;
import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.netty.ResponseFuture;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -62,6 +27,36 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.rocketmq.client.QueryResult;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.client.log.ClientLogger;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.protocol.NamespaceUtil;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.common.message.MessageClientIDSetter;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageId;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryMessageResponseHeader;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.InvokeCallback;
import org.apache.rocketmq.remoting.common.RemotingUtil;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.netty.ResponseFuture;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
public class MQAdminImpl {
private final InternalLogger log = ClientLogger.getLog();
@@ -187,21 +182,16 @@ public class MQAdminImpl {
}
public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {
LogicalQueueRouteData logicalQueueRouteData = searchLogicalQueueRouteByTimestamp(mq, timestamp);
if (logicalQueueRouteData != null) {
mq = logicalQueueRouteData.getMessageQueue();
}
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == brokerAddr) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (brokerAddr != null) {
try {
long offset = this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp,
return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp,
timeoutMillis);
return correctLogicalQueueOffset(offset, logicalQueueRouteData);
} catch (Exception e) {
throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
}
@@ -211,60 +201,15 @@ public class MQAdminImpl {
}
public long maxOffset(MessageQueue mq) throws MQClientException {
return this.maxOffset(mq, true);
}
public long maxOffset(MessageQueue mq, boolean committed) throws MQClientException {
final MessageQueue origMq = mq;
String topic = mq.getTopic();
LogicalQueueRouteData previousQueueRouteData = null;
for (int i = 0; i < 5; i++) {
LogicalQueueRouteData maxQueueRouteData = this.searchLogicalQueueRouteByOffset(origMq, Long.MAX_VALUE);
if (maxQueueRouteData != null) {
if (previousQueueRouteData != null && Objects.equal(previousQueueRouteData.getMessageQueue(), maxQueueRouteData.getMessageQueue())) {
throw new MQClientException("Topic route info not latest", null);
}
previousQueueRouteData = maxQueueRouteData;
mq = maxQueueRouteData.getMessageQueue();
}
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
if (null == brokerAddr) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
}
if (brokerAddr != null) {
try {
long offset = this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, topic, mq.getQueueId(), committed, maxQueueRouteData != null, timeoutMillis);
return correctLogicalQueueOffset(offset, maxQueueRouteData);
} catch (MQRedirectException e) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, false, null, Collections.singleton(mq.getQueueId()));
continue;
} catch (Exception e) {
throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
}
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
throw new MQClientException("Redirect exceed max times", null);
}
public long minOffset(MessageQueue mq) throws MQClientException {
LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L);
if (minQueueRouteData != null) {
mq = minQueueRouteData.getMessageQueue();
}
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == brokerAddr) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (brokerAddr != null) {
try {
long offset = this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis);
return correctLogicalQueueOffset(offset, minQueueRouteData);
return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis);
} catch (Exception e) {
throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
}
@@ -273,33 +218,29 @@ public class MQAdminImpl {
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
private List<LogicalQueueRouteData> queryLogicalQueueRouteData(MessageQueue mq) {
if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) {
TopicRouteData topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic());
if (topicRouteData == null) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
topicRouteData = this.mQClientFactory.queryTopicRouteData(mq.getTopic());
}
if (topicRouteData != null) {
LogicalQueuesInfo logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo();
if (logicalQueuesInfo != null) {
return logicalQueuesInfo.get(mq.getQueueId());
}
public long minOffset(MessageQueue mq) throws MQClientException {
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == brokerAddr) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (brokerAddr != null) {
try {
return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timeoutMillis);
} catch (Exception e) {
throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
}
}
return null;
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException {
LogicalQueueRouteData minQueueRouteData = searchLogicalQueueRouteByOffset(mq, 0L);
if (minQueueRouteData != null) {
mq = minQueueRouteData.getMessageQueue();
}
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
if (null == brokerAddr) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(this.mQClientFactory.getBrokerNameFromMessageQueue(mq));
}
if (brokerAddr != null) {
@@ -504,71 +445,4 @@ public class MQAdminImpl {
throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "The topic[" + topic + "] not matched route info");
}
private static long correctLogicalQueueOffset(long offset, LogicalQueueRouteData logicalQueueRouteData) {
if (logicalQueueRouteData == null) {
return offset;
}
return logicalQueueRouteData.toLogicalQueueOffset(offset);
}
private LogicalQueueRouteData searchLogicalQueueRouteByTimestamp(MessageQueue mq, long timestamp) {
List<LogicalQueueRouteData> queueRouteDataList = this.queryLogicalQueueRouteData(mq);
if (queueRouteDataList == null) {
return null;
}
LogicalQueueRouteData logicalQueueRouteData = null;
for (LogicalQueueRouteData el : queueRouteDataList) {
if (!el.isReadable()) {
continue;
}
if (logicalQueueRouteData == null && el.getFirstMsgTimeMillis() < 0) {
logicalQueueRouteData = el;
} else if (el.getFirstMsgTimeMillis() >= 0) {
if (el.getFirstMsgTimeMillis() <= timestamp && el.getLastMsgTimeMillis() >= timestamp) {
logicalQueueRouteData = el;
break;
}
}
}
if (logicalQueueRouteData == null) {
logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1);
}
return logicalQueueRouteData;
}
private LogicalQueueRouteData searchLogicalQueueRouteByOffset(MessageQueue mq, long offset) {
List<LogicalQueueRouteData> queueRouteDataList = this.queryLogicalQueueRouteData(mq);
if (queueRouteDataList == null) {
return null;
}
{
List<LogicalQueueRouteData> list = Lists.newArrayListWithCapacity(queueRouteDataList.size());
for (LogicalQueueRouteData queueRouteData : queueRouteDataList) {
if (LogicalQueueRouteData.READABLE_PREDICT.apply(queueRouteData)) {
list.add(queueRouteData);
}
}
queueRouteDataList = list;
}
if (queueRouteDataList.isEmpty()) {
return null;
}
if (offset <= 0) {
// min
return Collections.min(queueRouteDataList);
} else if (offset == Long.MAX_VALUE) {
// max
return Collections.max(queueRouteDataList);
}
Collections.sort(queueRouteDataList);
LogicalQueueRouteData searchKey = new LogicalQueueRouteData();
searchKey.setLogicalQueueDelta(offset);
int idx = Collections.binarySearch(queueRouteDataList, searchKey);
if (idx < 0) {
idx = -idx - 1;
idx -= 1;
}
return queueRouteDataList.get(idx);
}
}
client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+51 -188
View File
@@ -16,7 +16,6 @@
*/
package org.apache.rocketmq.client.impl;
import com.google.common.base.Function;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -42,7 +41,7 @@ import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.exception.OffsetNotFoundException;
import org.apache.rocketmq.client.hook.SendMessageContext;
import org.apache.rocketmq.client.impl.consumer.PullResultExt;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
@@ -82,13 +81,11 @@ import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult;
import org.apache.rocketmq.common.protocol.body.ConsumeStatsList;
import org.apache.rocketmq.common.protocol.body.ConsumerConnection;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.CreateMessageQueueForLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.GetConsumerStatusBody;
import org.apache.rocketmq.common.protocol.body.GroupList;
import org.apache.rocketmq.common.protocol.body.KVTable;
import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody;
import org.apache.rocketmq.common.protocol.body.MigrateLogicalQueueBody;
import org.apache.rocketmq.common.protocol.body.ProducerConnection;
import org.apache.rocketmq.common.protocol.body.QueryAssignmentRequestBody;
import org.apache.rocketmq.common.protocol.body.QueryAssignmentResponseBody;
@@ -98,13 +95,10 @@ import org.apache.rocketmq.common.protocol.body.QueryCorrectionOffsetBody;
import org.apache.rocketmq.common.protocol.body.QueueTimeSpan;
import org.apache.rocketmq.common.protocol.body.ResetOffsetBody;
import org.apache.rocketmq.common.protocol.body.SetMessageRequestModeRequestBody;
import org.apache.rocketmq.common.protocol.body.ReuseTopicLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.SealTopicLogicalQueueRequestBody;
import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody;
import org.apache.rocketmq.common.protocol.body.UpdateTopicLogicalQueueMappingRequestBody;
import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader;
import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader;
@@ -115,7 +109,6 @@ import org.apache.rocketmq.common.protocol.header.CreateAccessConfigRequestHeade
import org.apache.rocketmq.common.protocol.header.CreateTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteAccessConfigRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicLogicalQueueRequestHeader;
import org.apache.rocketmq.common.protocol.header.DeleteTopicRequestHeader;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil;
@@ -149,7 +142,6 @@ import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHea
import org.apache.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader;
import org.apache.rocketmq.common.protocol.header.QueryMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader;
import org.apache.rocketmq.common.protocol.header.QueryTopicLogicalQueueMappingRequestHeader;
import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.ResumeCheckHalfMessageRequestHeader;
import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader;
@@ -176,13 +168,10 @@ import org.apache.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerR
import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.protocol.route.TopicRouteDataNameSrv;
import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.InvokeCallback;
@@ -201,8 +190,6 @@ import org.apache.rocketmq.remoting.protocol.LanguageCode;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import static com.google.common.base.Optional.fromNullable;
public class MQClientAPIImpl {
private final static InternalLogger log = ClientLogger.getLog();
@@ -335,7 +322,7 @@ public class MQClientAPIImpl {
public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig,
final long timeoutMillis)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
throws RemotingException, InterruptedException, MQClientException {
CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();
requestHeader.setTopic(topicConfig.getTopicName());
requestHeader.setDefaultTopic(defaultTopic);
@@ -624,13 +611,9 @@ public class MQClientAPIImpl {
producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false);
} catch (Exception e) {
if (e instanceof MQRedirectException) {
sendCallback.onException(e);
} else {
producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
retryTimesWhenSendFailed, times, e, context, false, producer);
}
producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance,
retryTimesWhenSendFailed, times, e, context, false, producer);
}
} else {
producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true);
@@ -672,7 +655,7 @@ public class MQClientAPIImpl {
String retryBrokerName = brokerName;//by default, it will send to the same broker
if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send
MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName);
retryBrokerName = mqChosen.getBrokerName();
retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen);
}
String addr = instance.findBrokerAddressInPublish(retryBrokerName);
log.warn(String.format("async send msg by retry {} times. topic={}, brokerAddr={}, brokerName={}", tmp, msg.getTopic(), addr,
@@ -709,11 +692,6 @@ public class MQClientAPIImpl {
final RemotingCommand response,
final String addr
) throws MQBrokerException, RemotingCommandException {
HashMap<String, String> extFields = response.getExtFields();
if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) {
throw new MQRedirectException(response.getBody());
}
SendStatus sendStatus;
switch (response.getCode()) {
case ResponseCode.FLUSH_DISK_TIMEOUT: {
@@ -962,11 +940,6 @@ public class MQClientAPIImpl {
private PullResult processPullResponse(
final RemotingCommand response,
final String addr) throws MQBrokerException, RemotingCommandException {
HashMap<String, String> extFields = response.getExtFields();
if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) {
throw new MQRedirectException(response.getBody());
}
PullStatus pullStatus = PullStatus.NO_NEW_MSG;
switch (response.getCode()) {
case ResponseCode.SUCCESS:
@@ -981,6 +954,7 @@ public class MQClientAPIImpl {
case ResponseCode.PULL_OFFSET_MOVED:
pullStatus = PullStatus.OFFSET_ILLEGAL;
break;
default:
throw new MQBrokerException(response.getCode(), response.getRemark(), addr);
}
@@ -989,7 +963,7 @@ public class MQClientAPIImpl {
(PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class);
return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), responseHeader.getMinOffset(),
responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody());
responseHeader.getMaxOffset(), null, responseHeader.getSuggestWhichBrokerId(), response.getBody(), responseHeader.getOffsetDelta());
}
private PopResult processPopResponse(final String brokerName, final RemotingCommand response, String topic,
@@ -1132,28 +1106,15 @@ public class MQClientAPIImpl {
}
public long getMaxOffset(final String addr, final String topic, final int queueId, final long timeoutMillis)
throws RemotingException, MQBrokerException, InterruptedException {
return getMaxOffset(addr, topic, queueId, true, false, timeoutMillis);
}
public long getMaxOffset(final String addr, final String topic, final int queueId, boolean committed,
boolean fromLogicalQueue,
final long timeoutMillis)
throws RemotingException, MQBrokerException, InterruptedException {
GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader();
requestHeader.setTopic(topic);
requestHeader.setQueueId(queueId);
requestHeader.setCommitted(committed);
requestHeader.setLogicalQueue(fromLogicalQueue);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader);
RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
request, timeoutMillis);
assert response != null;
HashMap<String, String> extFields = response.getExtFields();
if (extFields != null && extFields.containsKey(MessageConst.PROPERTY_REDIRECT)) {
throw new MQRedirectException(response.getBody());
}
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
GetMaxOffsetResponseHeader responseHeader =
@@ -1258,9 +1219,11 @@ public class MQClientAPIImpl {
case ResponseCode.SUCCESS: {
QueryConsumerOffsetResponseHeader responseHeader =
(QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
return responseHeader.getOffset();
}
case ResponseCode.PULL_NOT_FOUND: {
throw new OffsetNotFoundException(response.getCode(), response.getRemark(), addr);
}
default:
break;
}
@@ -1649,15 +1612,8 @@ public class MQClientAPIImpl {
public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,
boolean allowTopicNotExist) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
return getTopicRouteInfoFromNameServer(topic, timeoutMillis, allowTopicNotExist, null);
}
public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis,
boolean allowTopicNotExist, Set<Integer> logicalQueueIdsFilter) throws MQClientException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader();
requestHeader.setTopic(topic);
requestHeader.setSysFlag(MessageSysFlag.LOGICAL_QUEUE_FLAG);
requestHeader.setLogicalQueueIdsFilter(logicalQueueIdsFilter);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINFO_BY_TOPIC, requestHeader);
@@ -1674,11 +1630,7 @@ public class MQClientAPIImpl {
case ResponseCode.SUCCESS: {
byte[] body = response.getBody();
if (body != null) {
return fromNullable(RemotingSerializable.decode(body, TopicRouteDataNameSrv.class)).transform(new Function<TopicRouteDataNameSrv, TopicRouteData>() {
@Override public TopicRouteData apply(TopicRouteDataNameSrv srv) {
return srv.toTopicRouteData();
}
}).orNull();
return TopicRouteData.decode(body, TopicRouteData.class);
}
}
default:
@@ -2609,147 +2561,58 @@ public class MQClientAPIImpl {
}
}
public LogicalQueuesInfo queryTopicLogicalQueue(String brokerAddr, String topic,
long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
QueryTopicLogicalQueueMappingRequestHeader requestHeader = new QueryTopicLogicalQueueMappingRequestHeader();
requestHeader.setTopic(topic);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader);
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
return RemotingSerializable.decode(response.getBody(), LogicalQueuesInfo.class);
}
public void updateTopicLogicalQueue(String brokerAddr, String topic, int queueId, int logicalQueueIndex,
long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING, null);
UpdateTopicLogicalQueueMappingRequestBody requestBody = new UpdateTopicLogicalQueueMappingRequestBody();
requestBody.setTopic(topic);
requestBody.setQueueId(queueId);
requestBody.setLogicalQueueIdx(logicalQueueIndex);
request.setBody(requestBody.encode());
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
}
public void deleteTopicLogicalQueueMapping(String brokerAddr, String topic, long timeoutMillis) throws MQBrokerException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
DeleteTopicLogicalQueueRequestHeader requestHeader = new DeleteTopicLogicalQueueRequestHeader();
requestHeader.setTopic(topic);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_LOGICAL_QUEUE_MAPPING, requestHeader);
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
}
public LogicalQueueRouteData sealTopicLogicalQueue(String brokerAddr, LogicalQueueRouteData queueRouteData, long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEAL_TOPIC_LOGICAL_QUEUE, null);
SealTopicLogicalQueueRequestBody requestBody = new SealTopicLogicalQueueRequestBody();
MessageQueue messageQueue = queueRouteData.getMessageQueue();
requestBody.setTopic(messageQueue.getTopic());
requestBody.setQueueId(messageQueue.getQueueId());
requestBody.setLogicalQueueIndex(queueRouteData.getLogicalQueueIndex());
request.setBody(requestBody.encode());
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class);
}
public LogicalQueueRouteData reuseTopicLogicalQueue(String brokerAddr, String topic, int queueId,
int logicalQueueIdx,
MessageQueueRouteState messageQueueRouteState, long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.REUSE_TOPIC_LOGICAL_QUEUE, null);
ReuseTopicLogicalQueueRequestBody requestBody = new ReuseTopicLogicalQueueRequestBody();
requestBody.setTopic(topic);
requestBody.setQueueId(queueId);
requestBody.setLogicalQueueIndex(logicalQueueIdx);
requestBody.setMessageQueueRouteState(messageQueueRouteState);
request.setBody(requestBody.encode());
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class);
}
public LogicalQueueRouteData createMessageQueueForLogicalQueue(String brokerAddr, String topic, int logicalQueueIdx,
MessageQueueRouteState messageQueueStatus,
long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE, null);
CreateMessageQueueForLogicalQueueRequestBody requestBody = new CreateMessageQueueForLogicalQueueRequestBody();
requestBody.setTopic(topic);
requestBody.setLogicalQueueIndex(logicalQueueIdx);
requestBody.setMessageQueueStatus(messageQueueStatus);
request.setBody(requestBody.encode());
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
return RemotingSerializable.decode(response.getBody(), LogicalQueueRouteData.class);
}
private MigrateLogicalQueueBody migrateTopicLogicalQueue(int requestCode, String brokerAddr,
LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData,
long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, null);
MigrateLogicalQueueBody requestBody = new MigrateLogicalQueueBody();
requestBody.setFromQueueRouteData(fromQueueRouteData);
requestBody.setToQueueRouteData(toQueueRouteData);
request.setBody(requestBody.encode());
RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, timeoutMillis);
assert response != null;
if (response.getCode() != ResponseCode.SUCCESS) {
throw new MQBrokerException(response.getCode(), response.getRemark());
}
return response.getBody() != null ? RemotingSerializable.decode(response.getBody(), MigrateLogicalQueueBody.class) : null;
}
public MigrateLogicalQueueBody migrateTopicLogicalQueuePrepare(String brokerAddr,
LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData,
long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis);
}
public MigrateLogicalQueueBody migrateTopicLogicalQueueCommit(String brokerAddr,
LogicalQueueRouteData fromQueueRouteData, LogicalQueueRouteData toQueueRouteData,
long timeoutMillis) throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
return migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis);
}
public void migrateTopicLogicalQueueNotify(String brokerAddr,
LogicalQueueRouteData fromQueueRouteData,
LogicalQueueRouteData toQueueRouteData,
long timeoutMillis) throws InterruptedException, MQBrokerException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException {
migrateTopicLogicalQueue(RequestCode.MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY, brokerAddr, fromQueueRouteData, toQueueRouteData, timeoutMillis);
}
public TopicConfig getTopicConfig(final String brokerAddr, String topic,
public TopicConfigAndQueueMapping getTopicConfig(final String brokerAddr, String topic,
long timeoutMillis) throws InterruptedException,
RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
GetTopicConfigRequestHeader header = new GetTopicConfigRequestHeader();
header.setTopic(topic);
header.setLo(true);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_CONFIG, header);
RemotingCommand response = this.remotingClient
.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), brokerAddr), request, timeoutMillis);
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
return RemotingSerializable.decode(response.getBody(), TopicConfig.class);
return RemotingSerializable.decode(response.getBody(), TopicConfigAndQueueMapping.class);
}
//should check the exist
case ResponseCode.TOPIC_NOT_EXIST: {
//should return null?
break;
}
default:
break;
}
throw new MQBrokerException(response.getCode(), response.getRemark());
}
public void createStaticTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, final TopicQueueMappingDetail topicQueueMappingDetail, boolean force,
final long timeoutMillis) throws RemotingException, InterruptedException, MQBrokerException {
CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader();
requestHeader.setTopic(topicConfig.getTopicName());
requestHeader.setDefaultTopic(defaultTopic);
requestHeader.setReadQueueNums(topicConfig.getReadQueueNums());
requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums());
requestHeader.setPerm(topicConfig.getPerm());
requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name());
requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag());
requestHeader.setOrder(topicConfig.isOrder());
requestHeader.setForce(force);
RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_STATIC_TOPIC, requestHeader);
request.setBody(topicQueueMappingDetail.encode());
RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
request, timeoutMillis);
assert response != null;
switch (response.getCode()) {
case ResponseCode.SUCCESS: {
return;
}
default:
break;
}
throw new MQBrokerException(response.getCode(), response.getRemark());
}
}
client/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java
+1 -1
View File
@@ -306,7 +306,7 @@ public class ConsumeMessageConcurrentlyService implements ConsumeMessageService
// Wrap topic with namespace before sending back message.
msg.setTopic(this.defaultMQPushConsumer.withNamespace(msg.getTopic()));
try {
this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue().getBrokerName());
this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, this.defaultMQPushConsumer.queueWithNamespace(context.getMessageQueue()));
return true;
} catch (Exception e) {
log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), e);
client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java
+20 -22
View File
@@ -16,25 +16,6 @@
*/
package org.apache.rocketmq.client.impl.consumer;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;
import org.apache.rocketmq.client.consumer.MessageQueueListener;
@@ -74,6 +55,26 @@ import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.exception.RemotingException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
public class DefaultLitePullConsumerImpl implements MQConsumerInner {
private final InternalLogger log = ClientLogger.getLog();
@@ -928,9 +929,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner {
null
);
this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);
if (pullResult instanceof PullResultWithLogicalQueues) {
pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt();
}
if (!this.consumeMessageHookList.isEmpty()) {
ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext();
consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace());
client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java
+6 -4
View File
@@ -265,9 +265,6 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
null
);
this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData);
if (pullResult instanceof PullResultWithLogicalQueues) {
pullResult = ((PullResultWithLogicalQueues) pullResult).getOrigPullResultExt();
}
//If namespace is not null , reset Topic without namespace.
this.resetTopic(pullResult.getMsgFoundList());
if (!this.consumeMessageHookList.isEmpty()) {
@@ -574,10 +571,15 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner {
this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway);
}
@Deprecated
public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
try {
String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName)
String destBrokerName = brokerName;
if (destBrokerName != null && destBrokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {
destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPullConsumer.queueWithNamespace(new MessageQueue(msg.getTopic(), msg.getBrokerName(), msg.getQueueId())));
}
String brokerAddr = (null != destBrokerName) ? this.mQClientFactory.findBrokerAddressInPublish(destBrokerName)
: RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());
if (UtilAll.isBlank(consumerGroup)) {
client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java
+62 -25
View File
@@ -72,6 +72,7 @@ import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.NamespaceUtil;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.body.ConsumeStatus;
import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo;
import org.apache.rocketmq.common.protocol.body.PopProcessQueueInfo;
@@ -297,6 +298,9 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
long offset = -1L;
try {
offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue());
if (offset < 0) {
throw new MQClientException(ResponseCode.SYSTEM_ERROR, "Unexpected offset " + offset);
}
} catch (Exception e) {
this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException);
log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e);
@@ -704,35 +708,55 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup());
}
@Deprecated
public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
sendMessageBack(msg, delayLevel, brokerName, null);
}
public void sendMessageBack(MessageExt msg, int delayLevel, final MessageQueue mq)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
sendMessageBack(msg, delayLevel, null, mq);
}
private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, final MessageQueue mq)
throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
try {
String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName)
: RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());
this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg,
this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes());
if ((brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX))
|| (mq != null && mq.getBrokerName().startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX))) {
sendMessageBackAsNormalMessage(msg);
} else {
String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName)
: RemotingHelper.parseSocketAddressAddr(msg.getStoreHost());
this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg,
this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes());
}
} catch (Exception e) {
log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e);
Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody());
String originMsgId = MessageAccessor.getOriginMessageId(msg);
MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);
newMsg.setFlag(msg.getFlag());
MessageAccessor.setProperties(newMsg, msg.getProperties());
MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());
MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1));
MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes()));
MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED);
newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());
this.mQClientFactory.getDefaultMQProducer().send(newMsg);
sendMessageBackAsNormalMessage(msg);
} finally {
msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQPushConsumer.getNamespace()));
}
}
private void sendMessageBackAsNormalMessage(MessageExt msg) throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody());
String originMsgId = MessageAccessor.getOriginMessageId(msg);
MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId);
newMsg.setFlag(msg.getFlag());
MessageAccessor.setProperties(newMsg, msg.getProperties());
MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic());
MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1));
MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes()));
MessageAccessor.clearProperty(newMsg, MessageConst.PROPERTY_TRANSACTION_PREPARED);
newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes());
this.mQClientFactory.getDefaultMQProducer().send(newMsg);
}
void ackAsync(MessageExt message, String consumerGroup) {
final String extraInfo = message.getProperty(MessageConst.PROPERTY_POP_CK);
@@ -743,15 +767,21 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
long queueOffset = ExtraInfoUtil.getQueueOffset(extraInfoStrs);
String topic = message.getTopic();
String desBrokerName = brokerName;
if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {
desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId)));
}
FindBrokerResult
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);
}
if (findBrokerResult == null) {
log.error("The broker[" + brokerName + "] not exist");
log.error("The broker[" + desBrokerName + "] not exist");
return;
}
@@ -784,11 +814,17 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo);
String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs);
int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs);
String desBrokerName = brokerName;
if (brokerName != null && brokerName.startsWith(MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX)) {
desBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(this.defaultMQPushConsumer.queueWithNamespace(new MessageQueue(topic, brokerName, queueId)));
}
FindBrokerResult
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(desBrokerName, MixAll.MASTER_ID, true);
}
if (findBrokerResult != null) {
ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader();
@@ -798,10 +834,11 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner {
requestHeader.setConsumerGroup(consumerGroup);
requestHeader.setExtraInfo(extraInfo);
requestHeader.setInvisibleTime(invisibleTime);
//here the broker should be polished
this.mQClientFactory.getMQClientAPIImpl().changeInvisibleTimeAsync(brokerName, findBrokerResult.getBrokerAddr(), requestHeader, ASYNC_TIMEOUT, callback);
return;
}
throw new MQClientException("The broker[" + brokerName + "] not exist", null);
throw new MQClientException("The broker[" + desBrokerName + "] not exist", null);
}
public int getMaxReconsumeTimes() {
client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java
+21 -318
View File
@@ -16,15 +16,12 @@
*/
package org.apache.rocketmq.client.impl.consumer;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Objects;
import org.apache.rocketmq.client.consumer.PopCallback;
import org.apache.rocketmq.client.consumer.PullCallback;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.hook.FilterMessageContext;
import org.apache.rocketmq.client.hook.FilterMessageHook;
import org.apache.rocketmq.client.impl.CommunicationMode;
@@ -40,12 +37,8 @@ import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.sysflag.PullSysFlag;
import org.apache.rocketmq.logging.InternalLogger;
@@ -53,16 +46,12 @@ import org.apache.rocketmq.remoting.exception.RemotingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import static com.google.common.base.Optional.fromNullable;
public class PullAPIWrapper {
private final InternalLogger log = ClientLogger.getLog();
private final MQClientInstance mQClientFactory;
@@ -83,36 +72,13 @@ public class PullAPIWrapper {
public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult,
final SubscriptionData subscriptionData) {
final PullResultExt pullResultExt = (PullResultExt) pullResult;
LogicalQueueRouteData queueRouteData = null;
PullResultWithLogicalQueues pullResultWithLogicalQueues = null;
if (pullResultExt instanceof PullResultWithLogicalQueues) {
pullResultWithLogicalQueues = (PullResultWithLogicalQueues) pullResultExt;
queueRouteData = pullResultWithLogicalQueues.getQueueRouteData();
}
if (queueRouteData != null) {
pullResultWithLogicalQueues.setOrigPullResultExt(new PullResultExt(pullResultExt.getPullStatus(),
queueRouteData.toLogicalQueueOffset(pullResultExt.getNextBeginOffset()),
queueRouteData.toLogicalQueueOffset(pullResultExt.getMinOffset()),
// although this maxOffset may not belong to this queue route, but the actual value must be a larger one, and since maxOffset here is not an accurate value, we just do it to make things simple.
queueRouteData.toLogicalQueueOffset(pullResultExt.getMaxOffset()),
pullResultExt.getMsgFoundList(),
pullResultExt.getSuggestWhichBrokerId(),
pullResultExt.getMessageBinary()));
}
PullResultExt pullResultExt = (PullResultExt) pullResult;
this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
if (PullStatus.FOUND == pullResult.getPullStatus()) {
ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());
List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);
if (queueRouteData != null) {
// prevent pulled data is out of current queue route, this happens when some commit log data is cleaned in the broker but still pull from it.
msgList = queueRouteData.filterMessages(msgList);
}
List<MessageExt> msgListFilterAgain = msgList;
if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) {
msgListFilterAgain = new ArrayList<MessageExt>(msgList.size());
@@ -143,8 +109,8 @@ public class PullAPIWrapper {
Long.toString(pullResult.getMaxOffset()));
msg.setBrokerName(mq.getBrokerName());
msg.setQueueId(mq.getQueueId());
if (queueRouteData != null) {
msg.setQueueOffset(queueRouteData.toLogicalQueueOffset(msg.getQueueOffset()));
if (pullResultExt.getOffsetDelta() != null) {
msg.setQueueOffset(pullResultExt.getOffsetDelta() + msg.getQueueOffset());
}
}
@@ -153,7 +119,7 @@ public class PullAPIWrapper {
pullResultExt.setMessageBinary(null);
return pullResultExt;
return pullResult;
}
public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) {
@@ -182,72 +148,30 @@ public class PullAPIWrapper {
}
public PullResult pullKernelImpl(
MessageQueue mq,
final MessageQueue mq,
final String subExpression,
final String expressionType,
final long subVersion,
long offset,
final long offset,
final int maxNums,
final int sysFlag,
long commitOffset,
final long commitOffset,
final long brokerSuspendMaxTimeMillis,
final long timeoutMillis,
final CommunicationMode communicationMode,
PullCallback pullCallback
final PullCallback pullCallback
) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) {
LogicalQueueContext logicalQueueContext = new LogicalQueueContext(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback);
while (true) {
try {
MessageQueue messageQueue = logicalQueueContext.getModifiedMessageQueue();
if (messageQueue == null) {
if (pullCallback != null) {
pullCallback.onSuccess(logicalQueueContext.getPullResult());
return null;
} else {
return logicalQueueContext.getPullResult();
}
}
PullResult pullResult = this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, logicalQueueContext.getModifiedOffset(), maxNums, sysFlag, logicalQueueContext.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, logicalQueueContext.wrapPullCallback());
return logicalQueueContext.wrapPullResult(pullResult);
} catch (MQRedirectException e) {
if (!logicalQueueContext.shouldRetry(e)) {
throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect");
}
}
}
} else {
return this.pullKernelImplWithoutRetry(mq, subExpression, expressionType, subVersion, offset, maxNums, sysFlag, commitOffset, brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, pullCallback);
}
}
public PullResult pullKernelImplWithoutRetry(
MessageQueue mq,
final String subExpression,
final String expressionType,
final long subVersion,
long offset,
final int maxNums,
final int sysFlag,
long commitOffset,
final long brokerSuspendMaxTimeMillis,
final long timeoutMillis,
final CommunicationMode communicationMode,
PullCallback pullCallback
) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
String topic = mq.getTopic();
int queueId = mq.getQueueId();
FindBrokerResult findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),
this.recalculatePullFromWhichNode(mq), false);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic);
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult =
this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(),
this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq),
this.recalculatePullFromWhichNode(mq), false);
}
if (findBrokerResult != null) {
{
// check version
@@ -265,8 +189,8 @@ public class PullAPIWrapper {
PullMessageRequestHeader requestHeader = new PullMessageRequestHeader();
requestHeader.setConsumerGroup(this.consumerGroup);
requestHeader.setTopic(topic);
requestHeader.setQueueId(queueId);
requestHeader.setTopic(mq.getTopic());
requestHeader.setQueueId(mq.getQueueId());
requestHeader.setQueueOffset(offset);
requestHeader.setMaxMsgNums(maxNums);
requestHeader.setSysFlag(sysFlagInner);
@@ -278,15 +202,18 @@ public class PullAPIWrapper {
String brokerAddr = findBrokerResult.getBrokerAddr();
if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) {
brokerAddr = computePullFromWhichFilterServer(topic, brokerAddr);
brokerAddr = computePullFromWhichFilterServer(mq.getTopic(), brokerAddr);
}
return this.mQClientFactory.getMQClientAPIImpl().pullMessage(
PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(
brokerAddr,
requestHeader,
timeoutMillis,
communicationMode,
pullCallback);
return pullResult;
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
@@ -373,10 +300,10 @@ public class PullAPIWrapper {
public void popAsync(MessageQueue mq, long invisibleTime, int maxNums, String consumerGroup,
long timeout, PopCallback popCallback, boolean poll, int initMode, boolean order, String expressionType, String expression)
throws MQClientException, RemotingException, InterruptedException {
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);
if (null == findBrokerResult) {
this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);
}
if (findBrokerResult != null) {
PopMessageRequestHeader requestHeader = new PopMessageRequestHeader();
@@ -403,228 +330,4 @@ public class PullAPIWrapper {
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
private class LogicalQueueContext implements PullCallback {
private final MessageQueue mq;
private final String subExpression;
private final String expressionType;
private final long subVersion;
private final long offset;
private final int maxNums;
private final int sysFlag;
private final long commitOffset;
private final long brokerSuspendMaxTimeMillis;
private final long timeoutMillis;
private final CommunicationMode communicationMode;
private final PullCallback pullCallback;
private volatile LogicalQueuesInfo logicalQueuesInfo;
private volatile LogicalQueueRouteData logicalQueueRouteData;
private volatile boolean isMaxReadableQueueRoute;
private volatile PullResultExt pullResult = null;
private final AtomicInteger retry = new AtomicInteger();
public LogicalQueueContext(MessageQueue mq, String subExpression, String expressionType, long subVersion,
long offset, int maxNums, int sysFlag, long commitOffset, long brokerSuspendMaxTimeMillis,
long timeoutMillis, CommunicationMode communicationMode,
PullCallback pullCallback) {
this.mq = mq;
this.subExpression = subExpression;
this.expressionType = expressionType;
this.subVersion = subVersion;
this.offset = offset;
this.maxNums = maxNums;
this.sysFlag = sysFlag;
this.commitOffset = commitOffset;
this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis;
this.timeoutMillis = timeoutMillis;
this.communicationMode = communicationMode;
this.pullCallback = pullCallback;
this.buildLogicalQueuesInfo();
}
private boolean notUsingLogicalQueue() {
return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null;
}
private void buildLogicalQueuesInfo() {
TopicRouteData topicRouteData = PullAPIWrapper.this.mQClientFactory.queryTopicRouteData(mq.getTopic());
if (topicRouteData != null) {
this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo();
}
}
@Override public void onSuccess(PullResult pullResult) {
this.pullCallback.onSuccess(this.wrapPullResult(pullResult));
}
@Override public void onException(Throwable t) {
if (!this.shouldRetry(t)) {
this.pullCallback.onException(t);
return;
}
MessageQueue messageQueue = this.getModifiedMessageQueue();
if (messageQueue == null) {
this.pullCallback.onSuccess(this.getPullResult());
return;
}
try {
PullAPIWrapper.this.pullKernelImplWithoutRetry(messageQueue, subExpression, expressionType, subVersion, this.getModifiedOffset(), maxNums, sysFlag, this.getModifiedCommitOffset(), brokerSuspendMaxTimeMillis, timeoutMillis, communicationMode, this);
} catch (Exception e) {
this.pullCallback.onException(e);
}
}
public MessageQueue getModifiedMessageQueue() {
if (this.notUsingLogicalQueue()) {
return this.mq;
}
this.logicalQueuesInfo.readLock().lock();
try {
List<LogicalQueueRouteData> queueRouteDataList = fromNullable(this.logicalQueuesInfo.get(this.mq.getQueueId())).or(Collections.<LogicalQueueRouteData>emptyList());
LogicalQueueRouteData searchKey = new LogicalQueueRouteData();
searchKey.setState(MessageQueueRouteState.Normal);
searchKey.setLogicalQueueDelta(offset);
// it's sorted after getTopicRouteInfoFromNameServer
int startIdx = Collections.binarySearch(queueRouteDataList, searchKey);
if (startIdx < 0) {
startIdx = -startIdx - 1;
// lower entry
startIdx -= 1;
}
this.logicalQueueRouteData = null;
this.pullResult = null;
LogicalQueueRouteData lastReadableLogicalQueueRouteData = null; // first item which delta > offset
LogicalQueueRouteData minReadableLogicalQueueRouteData = null;
LogicalQueueRouteData maxReadableLogicalQueueRouteData = null;
for (int i = 0, size = queueRouteDataList.size(); i < size; i++) {
LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i);
if (!queueRouteData.isReadable()) {
continue;
}
maxReadableLogicalQueueRouteData = queueRouteData;
if (minReadableLogicalQueueRouteData == null) {
minReadableLogicalQueueRouteData = queueRouteData;
if (i < startIdx) {
// must consider following `i++` operation when invoke `continue`, so decrease first
i = startIdx - 1;
continue;
}
}
if (queueRouteData.getLogicalQueueDelta() > offset) {
if (this.logicalQueueRouteData != null) {
if (this.logicalQueueRouteData.toLogicalQueueOffset(this.logicalQueueRouteData.getOffsetMax()) <= offset) {
this.logicalQueueRouteData = queueRouteData;
}
break;
} else {
if (lastReadableLogicalQueueRouteData == null) {
lastReadableLogicalQueueRouteData = queueRouteData;
}
}
} else {
this.logicalQueueRouteData = queueRouteData;
}
}
if (this.logicalQueueRouteData == null) {
if (lastReadableLogicalQueueRouteData != null) {
this.pullResult = new PullResultExt(PullStatus.OFFSET_ILLEGAL, lastReadableLogicalQueueRouteData.getLogicalQueueDelta(), minReadableLogicalQueueRouteData.getLogicalQueueDelta(), maxReadableLogicalQueueRouteData.getLogicalQueueDelta(), null, 0, null);
return null;
} else {
if (maxReadableLogicalQueueRouteData != null) {
this.logicalQueueRouteData = maxReadableLogicalQueueRouteData;
} else {
if (!queueRouteDataList.isEmpty()) {
this.logicalQueueRouteData = queueRouteDataList.get(queueRouteDataList.size() - 1);
} else {
pullResult = new PullResultExt(PullStatus.NO_NEW_MSG, 0, 0, 0, null, 0, null);
return null;
}
}
}
}
this.isMaxReadableQueueRoute = this.logicalQueueRouteData.isSameTo(maxReadableLogicalQueueRouteData);
return this.logicalQueueRouteData.getMessageQueue();
} finally {
this.logicalQueuesInfo.readLock().unlock();
}
}
public PullResultExt getPullResult() {
return pullResult;
}
public PullCallback wrapPullCallback() {
if (this.notUsingLogicalQueue()) {
return this.pullCallback;
}
if (!CommunicationMode.ASYNC.equals(this.communicationMode)) {
return this.pullCallback;
}
return this;
}
public long getModifiedOffset() {
return this.logicalQueueRouteData.toMessageQueueOffset(this.offset);
}
public long getModifiedCommitOffset() {
// TODO should this be modified too? If offset is not in current broker's range, how do we handle it?
return this.commitOffset;
}
public void incrRetry() {
this.retry.incrementAndGet();
}
public boolean shouldRetry(Throwable t) {
this.incrRetry();
if (this.retry.get() >= 3) {
return false;
}
if (t instanceof MQRedirectException) {
MQRedirectException e = (MQRedirectException) t;
this.processResponseBody(e.getBody());
return true;
}
return false;
}
public PullResult wrapPullResult(PullResult pullResult) {
if (pullResult == null) {
return null;
}
if (this.logicalQueueRouteData == null) {
return pullResult;
}
if (!this.isMaxReadableQueueRoute && PullStatus.NO_MATCHED_MSG.equals(pullResult.getPullStatus())) {
PullStatus status = PullStatus.OFFSET_ILLEGAL;
if (pullResult instanceof PullResultExt) {
PullResultExt pullResultExt = (PullResultExt) pullResult;
pullResult = new PullResultExt(status, pullResultExt.getNextBeginOffset(), pullResultExt.getMinOffset(), pullResultExt.getMaxOffset(), pullResultExt.getMsgFoundList(), pullResultExt.getSuggestWhichBrokerId(), pullResultExt.getMessageBinary());
} else {
pullResult = new PullResult(status, pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList());
}
}
// method PullAPIWrapper#processPullResult will modify queueOffset/nextBeginOffset/minOffset/maxOffset
return new PullResultWithLogicalQueues(pullResult, this.logicalQueueRouteData);
}
public void processResponseBody(byte[] responseBody) {
log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.logicalQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null);
if (responseBody != null) {
try {
List<LogicalQueueRouteData> queueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA);
this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), queueRouteDataList);
return;
} catch (Exception e) {
log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.logicalQueueRouteData, e);
}
}
PullAPIWrapper.this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic(), false, null, Collections.singleton(this.mq.getQueueId()));
this.buildLogicalQueuesInfo();
}
}
}
client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultExt.java
+12
View File
@@ -25,11 +25,23 @@ public class PullResultExt extends PullResult {
private final long suggestWhichBrokerId;
private byte[] messageBinary;
private final Long offsetDelta;
public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,
List<MessageExt> msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary) {
this(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList, suggestWhichBrokerId, messageBinary, 0L);
}
public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset,
List<MessageExt> msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary, final Long offsetDelta) {
super(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList);
this.suggestWhichBrokerId = suggestWhichBrokerId;
this.messageBinary = messageBinary;
this.offsetDelta = offsetDelta;
}
public Long getOffsetDelta() {
return offsetDelta;
}
public byte[] getMessageBinary() {
client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullResultWithLogicalQueues.java
-96
View File
@@ -1,96 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.client.impl.consumer;
import java.util.List;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.client.consumer.PullStatus;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
public class PullResultWithLogicalQueues extends PullResultExt {
private PullResultExt origPullResultExt;
private final LogicalQueueRouteData queueRouteData;
public PullResultWithLogicalQueues(PullResult pullResult, LogicalQueueRouteData floorQueueRouteData) {
super(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(),
pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getSuggestWhichBrokerId() : MixAll.MASTER_ID,
pullResult instanceof PullResultExt ? ((PullResultExt) pullResult).getMessageBinary() : null);
if (pullResult instanceof PullResultExt) {
this.origPullResultExt = (PullResultExt) pullResult;
} else {
this.origPullResultExt = new PullResultExt(pullResult.getPullStatus(), pullResult.getNextBeginOffset(), pullResult.getMinOffset(), pullResult.getMaxOffset(), pullResult.getMsgFoundList(), MixAll.MASTER_ID, null);
}
this.queueRouteData = floorQueueRouteData;
}
public PullResult getOrigPullResultExt() {
return origPullResultExt;
}
public LogicalQueueRouteData getQueueRouteData() {
return queueRouteData;
}
public void setOrigPullResultExt(PullResultExt pullResultExt) {
this.origPullResultExt = pullResultExt;
}
@Override public PullStatus getPullStatus() {
return origPullResultExt.getPullStatus();
}
@Override public long getNextBeginOffset() {
return origPullResultExt.getNextBeginOffset();
}
@Override public long getMinOffset() {
return origPullResultExt.getMinOffset();
}
@Override public long getMaxOffset() {
return origPullResultExt.getMaxOffset();
}
@Override public List<MessageExt> getMsgFoundList() {
return origPullResultExt.getMsgFoundList();
}
@Override public void setMsgFoundList(List<MessageExt> msgFoundList) {
origPullResultExt.setMsgFoundList(msgFoundList);
}
@Override public byte[] getMessageBinary() {
return origPullResultExt.getMessageBinary();
}
@Override public void setMessageBinary(byte[] messageBinary) {
origPullResultExt.setMessageBinary(messageBinary);
}
@Override public long getSuggestWhichBrokerId() {
return origPullResultExt.getSuggestWhichBrokerId();
}
@Override public String toString() {
return "PullResultWithLogicalQueues{" +
"origPullResultExt=" + origPullResultExt +
", queueRouteData=" + queueRouteData +
'}';
}
}
client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java
+4 -3
View File
@@ -76,7 +76,7 @@ public abstract class RebalanceImpl {
}
public void unlock(final MessageQueue mq, final boolean oneway) {
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);
if (findBrokerResult != null) {
UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody();
requestBody.setConsumerGroup(this.consumerGroup);
@@ -141,7 +141,8 @@ public abstract class RebalanceImpl {
continue;
}
Set<MessageQueue> mqs = result.get(mq.getBrokerName());
String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);
Set<MessageQueue> mqs = result.get(destBrokerName);
if (null == mqs) {
mqs = new HashSet<MessageQueue>();
result.put(mq.getBrokerName(), mqs);
@@ -154,7 +155,7 @@ public abstract class RebalanceImpl {
}
public boolean lock(final MessageQueue mq) {
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true);
FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(this.mQClientFactory.getBrokerNameFromMessageQueue(mq), MixAll.MASTER_ID, true);
if (findBrokerResult != null) {
LockBatchRequestBody requestBody = new LockBatchRequestBody();
requestBody.setConsumerGroup(this.consumerGroup);
client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java
+12 -3
View File
@@ -29,6 +29,7 @@ import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.constant.ConsumeInitMode;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
@@ -188,7 +189,8 @@ public class RebalancePushImpl extends RebalanceImpl {
}
}
} else {
result = -1;
throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query consume offset from " +
"offset store");
}
break;
}
@@ -197,9 +199,11 @@ public class RebalancePushImpl extends RebalanceImpl {
if (lastOffset >= 0) {
result = lastOffset;
} else if (-1 == lastOffset) {
//the offset will be fixed by the OFFSET_ILLEGAL process
result = 0L;
} else {
result = -1;
throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query offset from offset " +
"store");
}
break;
}
@@ -226,7 +230,8 @@ public class RebalancePushImpl extends RebalanceImpl {
}
}
} else {
result = -1;
throw new MQClientException(ResponseCode.QUERY_NOT_FOUND, "Failed to query offset from offset " +
"store");
}
break;
}
@@ -235,6 +240,10 @@ public class RebalancePushImpl extends RebalanceImpl {
break;
}
if (result < 0) {
throw new MQClientException(ResponseCode.SYSTEM_ERROR, "Found unexpected result " + result);
}
return result;
}
client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java
+71 -98
View File
@@ -16,26 +16,6 @@
*/
package org.apache.rocketmq.client.impl.factory;
import java.io.UnsupportedEncodingException;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.admin.MQAdminExtInner;
@@ -77,8 +57,6 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
import org.apache.rocketmq.common.protocol.heartbeat.ProducerData;
import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.logging.InternalLogger;
@@ -88,10 +66,32 @@ import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.netty.NettyClientConfig;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import java.io.UnsupportedEncodingException;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import static org.apache.rocketmq.common.rpc.ClientMetadata.topicRouteData2EndpointsForStaticTopic;
public class MQClientInstance {
private final static long LOCK_TIMEOUT_MILLIS = 3000;
private final InternalLogger log = ClientLogger.getLog();
private final static InternalLogger log = ClientLogger.getLog();
private final ClientConfig clientConfig;
private final int instanceIndex;
private final String clientId;
@@ -103,6 +103,7 @@ public class MQClientInstance {
private final MQClientAPIImpl mQClientAPIImpl;
private final MQAdminImpl mQAdminImpl;
private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<String, TopicRouteData>();
private final ConcurrentMap<String/* Topic */, ConcurrentMap<MessageQueue, String/*brokerName*/>> topicEndPointsTable = new ConcurrentHashMap<>();
private final Lock lockNamesrv = new ReentrantLock();
private final Lock lockHeartbeat = new ReentrantLock();
private final ConcurrentMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =
@@ -162,8 +163,12 @@ public class MQClientInstance {
MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION), RemotingCommand.getSerializeTypeConfigInThisServer());
}
public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topic, final TopicRouteData route) {
TopicPublishInfo info = new TopicPublishInfo();
// TO DO should check the usage of raw route, it is better to remove such field
info.setTopicRouteData(route);
if (route.getOrderTopicConf() != null && route.getOrderTopicConf().length() > 0) {
String[] brokers = route.getOrderTopicConf().split(";");
@@ -177,28 +182,13 @@ public class MQClientInstance {
}
info.setOrderTopic(true);
} else if (route.getOrderTopicConf() == null && route.getLogicalQueuesInfo() != null) {
} else if (route.getOrderTopicConf() == null
&& route.getTopicQueueMappingByBroker() != null
&& !route.getTopicQueueMappingByBroker().isEmpty()) {
info.setOrderTopic(false);
List<MessageQueue> messageQueueList = info.getMessageQueueList();
LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo();
for (Map.Entry<Integer, List<LogicalQueueRouteData>> entry : logicalQueueInfo.entrySet()) {
boolean someWritable = false;
for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) {
if (logicalQueueRouteData.isWritable()) {
someWritable = true;
break;
}
}
if (!someWritable) {
continue;
}
MessageQueue mq = new MessageQueue();
mq.setQueueId(entry.getKey());
mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME);
mq.setTopic(topic);
messageQueueList.add(mq);
}
Collections.sort(messageQueueList, new Comparator<MessageQueue>() {
ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route);
info.getMessageQueueList().addAll(mqEndPoints.keySet());
Collections.sort(info.getMessageQueueList(), new Comparator<MessageQueue>() {
@Override public int compare(MessageQueue o1, MessageQueue o2) {
return MixAll.compareInteger(o1.getQueueId(), o2.getQueueId());
}
@@ -239,26 +229,10 @@ public class MQClientInstance {
public static Set<MessageQueue> topicRouteData2TopicSubscribeInfo(final String topic, final TopicRouteData route) {
Set<MessageQueue> mqList = new HashSet<MessageQueue>();
if (route.getLogicalQueuesInfo() != null) {
LogicalQueuesInfo logicalQueueInfo = route.getLogicalQueuesInfo();
for (Map.Entry<Integer, List<LogicalQueueRouteData>> entry : logicalQueueInfo.entrySet()) {
boolean someReadable = false;
for (LogicalQueueRouteData logicalQueueRouteData : entry.getValue()) {
if (logicalQueueRouteData.isReadable()) {
someReadable = true;
break;
}
}
if (!someReadable) {
continue;
}
MessageQueue mq = new MessageQueue();
mq.setQueueId(entry.getKey());
mq.setBrokerName(MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME);
mq.setTopic(topic);
mqList.add(mq);
}
return mqList;
if (route.getTopicQueueMappingByBroker() != null
&& !route.getTopicQueueMappingByBroker().isEmpty()) {
ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, route);
return mqEndPoints.keySet();
}
List<QueueData> qds = route.getQueueDatas();
for (QueueData qd : qds) {
@@ -656,11 +630,6 @@ public class MQClientInstance {
public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault,
DefaultMQProducer defaultMQProducer) {
return this.updateTopicRouteInfoFromNameServer(topic, isDefault, defaultMQProducer, null);
}
public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault,
DefaultMQProducer defaultMQProducer, Set<Integer> logicalQueueIdsFilter) {
try {
if (this.lockNamesrv.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) {
try {
@@ -676,7 +645,7 @@ public class MQClientInstance {
}
}
} else {
topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout(), true, logicalQueueIdsFilter);
topicRouteData = this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, clientConfig.getMqClientApiTimeout());
}
if (topicRouteData != null) {
TopicRouteData old = this.topicRouteTable.get(topic);
@@ -688,31 +657,19 @@ public class MQClientInstance {
}
if (changed) {
TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData);
if (logicalQueueIdsFilter != null && cloneTopicRouteData.getLogicalQueuesInfo() != null) {
TopicRouteData curTopicRouteData = this.topicRouteTable.get(topic);
if (curTopicRouteData != null) {
LogicalQueuesInfo curLogicalQueuesInfo = curTopicRouteData.getLogicalQueuesInfo();
if (curLogicalQueuesInfo != null) {
LogicalQueuesInfo cloneLogicalQueuesInfo = cloneTopicRouteData.getLogicalQueuesInfo();
curLogicalQueuesInfo.readLock().lock();
try {
for (Entry<Integer, List<LogicalQueueRouteData>> entry : curLogicalQueuesInfo.entrySet()) {
if (!cloneLogicalQueuesInfo.containsKey(entry.getKey())) {
cloneLogicalQueuesInfo.put(entry.getKey(), entry.getValue());
}
}
} finally {
curLogicalQueuesInfo.readLock().unlock();
}
}
}
}
for (BrokerData bd : topicRouteData.getBrokerDatas()) {
this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());
}
// Update endpoint map
{
ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData);
if (mqEndPoints != null && !mqEndPoints.isEmpty()) {
topicEndPointsTable.put(topic, mqEndPoints);
}
}
// Update Pub info
{
TopicPublishInfo publishInfo = topicRouteData2TopicPublishInfo(topic, topicRouteData);
@@ -739,6 +696,7 @@ public class MQClientInstance {
}
}
}
TopicRouteData cloneTopicRouteData = new TopicRouteData(topicRouteData);
log.info("topicRouteTable.put. Topic = {}, TopicRouteData[{}]", topic, cloneTopicRouteData);
this.topicRouteTable.put(topic, cloneTopicRouteData);
return true;
@@ -865,13 +823,6 @@ public class MQClientInstance {
private boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) {
if (olddata == null || nowdata == null)
return true;
LogicalQueuesInfo oldLogicalQueuesInfo = olddata.getLogicalQueuesInfo();
LogicalQueuesInfo newLogicalQueuesInfo = nowdata.getLogicalQueuesInfo();
if (oldLogicalQueuesInfo != null && newLogicalQueuesInfo != null) {
return oldLogicalQueuesInfo.keySet().equals(newLogicalQueuesInfo.keySet());
} else if (oldLogicalQueuesInfo != null || newLogicalQueuesInfo != null) {
return true;
}
TopicRouteData old = new TopicRouteData(olddata);
TopicRouteData now = new TopicRouteData(nowdata);
Collections.sort(old.getQueueDatas());
@@ -1067,7 +1018,21 @@ public class MQClientInstance {
return this.consumerTable.get(group);
}
public String getBrokerNameFromMessageQueue(final MessageQueue mq) {
if (topicEndPointsTable != null
&& topicEndPointsTable.get(mq.getTopic()) != null
&& !topicEndPointsTable.get(mq.getTopic()).isEmpty()) {
return topicEndPointsTable.get(mq.getTopic()).get(mq);
}
return mq.getBrokerName();
}
public FindBrokerResult findBrokerAddressInAdmin(final String brokerName) {
if (brokerName == null) {
return null;
}
String brokerAddr = null;
boolean slave = false;
boolean found = false;
@@ -1097,7 +1062,11 @@ public class MQClientInstance {
return null;
}
public String findBrokerAddressInPublish(final String brokerName) {
if (brokerName == null) {
return null;
}
HashMap<Long/* brokerId */, String/* address */> map = this.brokerAddrTable.get(brokerName);
if (map != null && !map.isEmpty()) {
return map.get(MixAll.MASTER_ID);
@@ -1106,11 +1075,15 @@ public class MQClientInstance {
return null;
}
public FindBrokerResult findBrokerAddressInSubscribe(
final String brokerName,
final long brokerId,
final boolean onlyThisBroker
) {
if (brokerName == null) {
return null;
}
String brokerAddr = null;
boolean slave = false;
boolean found = false;
@@ -1141,7 +1114,7 @@ public class MQClientInstance {
return null;
}
public int findBrokerVersion(String brokerName, String brokerAddr) {
private int findBrokerVersion(String brokerName, String brokerAddr) {
if (this.brokerVersionTable.containsKey(brokerName)) {
if (this.brokerVersionTable.get(brokerName).containsKey(brokerAddr)) {
return this.brokerVersionTable.get(brokerName).get(brokerAddr);
client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java
+9 -227
View File
@@ -16,16 +16,12 @@
*/
package org.apache.rocketmq.client.impl.producer;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Objects;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
@@ -45,7 +41,6 @@ import org.apache.rocketmq.client.Validators;
import org.apache.rocketmq.client.common.ClientErrorCode;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.exception.RequestTimeoutException;
import org.apache.rocketmq.client.hook.CheckForbiddenContext;
import org.apache.rocketmq.client.hook.CheckForbiddenHook;
@@ -67,7 +62,6 @@ import org.apache.rocketmq.client.producer.RequestFutureTable;
import org.apache.rocketmq.client.producer.RequestResponseFuture;
import org.apache.rocketmq.client.producer.SendCallback;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.client.producer.SendResultForLogicalQueue;
import org.apache.rocketmq.client.producer.SendStatus;
import org.apache.rocketmq.client.producer.TransactionCheckListener;
import org.apache.rocketmq.client.producer.TransactionListener;
@@ -88,13 +82,9 @@ import org.apache.rocketmq.common.message.MessageId;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.message.MessageType;
import org.apache.rocketmq.common.protocol.NamespaceUtil;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader;
import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.sysflag.MessageSysFlag;
import org.apache.rocketmq.common.utils.CorrelationIdUtil;
import org.apache.rocketmq.logging.InternalLogger;
@@ -730,43 +720,13 @@ public class DefaultMQProducerImpl implements MQProducerInner {
final SendCallback sendCallback,
final TopicPublishInfo topicPublishInfo,
final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
if (MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME.equals(mq.getBrokerName())) {
LogicalQueueSendContext logicalQueueContext = new LogicalQueueSendContext(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout);
while (true) {
try {
SendResult sendResult = this.sendKernelImplWithoutRetry(msg,
logicalQueueContext.getModifiedMessageQueue(),
communicationMode,
logicalQueueContext.wrapSendCallback(),
topicPublishInfo,
timeout);
return logicalQueueContext.wrapSendResult(sendResult);
} catch (MQRedirectException e) {
if (!logicalQueueContext.shouldRetry(e)) {
throw new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect");
}
} catch (RemotingException e) {
if (!logicalQueueContext.shouldRetry(e)) {
throw e;
}
}
}
} else {
return sendKernelImplWithoutRetry(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout);
}
}
private SendResult sendKernelImplWithoutRetry(final Message msg,
final MessageQueue mq,
final CommunicationMode communicationMode,
SendCallback sendCallback,
final TopicPublishInfo topicPublishInfo,
final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException {
long beginStartTime = System.currentTimeMillis();
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
String brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);
String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName);
if (null == brokerAddr) {
tryToFindTopicPublishInfo(mq.getTopic());
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
brokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(mq);
brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(brokerName);
}
SendMessageContext context = null;
@@ -798,10 +758,6 @@ public class DefaultMQProducerImpl implements MQProducerInner {
sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE;
}
if (!CommunicationMode.ONEWAY.equals(communicationMode)) {
sysFlag |= MessageSysFlag.LOGICAL_QUEUE_FLAG;
}
if (hasCheckForbiddenHook()) {
CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext();
checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr());
@@ -890,7 +846,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
brokerName,
tmpMessage,
requestHeader,
timeout - costTimeAsync,
@@ -910,7 +866,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
}
sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(
brokerAddr,
mq.getBrokerName(),
brokerName,
msg,
requestHeader,
timeout - costTimeSync,
@@ -941,7 +897,7 @@ public class DefaultMQProducerImpl implements MQProducerInner {
}
}
throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
throw new MQClientException("The broker[" + brokerName + "] not exist", null);
}
public MQClientInstance getmQClientFactory() {
@@ -1042,7 +998,6 @@ public class DefaultMQProducerImpl implements MQProducerInner {
executeEndTransactionHook(context);
}
}
/**
* DEFAULT ONEWAY -------------------------------------------------------
*/
@@ -1377,7 +1332,8 @@ public class DefaultMQProducerImpl implements MQProducerInner {
id = MessageDecoder.decodeMessageId(sendResult.getMsgId());
}
String transactionId = sendResult.getTransactionId();
final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName());
final String destBrokerName = this.mQClientFactory.getBrokerNameFromMessageQueue(defaultMQProducer.queueWithNamespace(sendResult.getMessageQueue()));
final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(destBrokerName);
EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader();
requestHeader.setTransactionId(transactionId);
requestHeader.setCommitLogOffset(id.getOffset());
@@ -1682,178 +1638,4 @@ public class DefaultMQProducerImpl implements MQProducerInner {
public DefaultMQProducer getDefaultMQProducer() {
return defaultMQProducer;
}
private class LogicalQueueSendContext implements SendCallback {
private final Message msg;
private final MessageQueue mq;
private final CommunicationMode communicationMode;
private final SendCallback sendCallback;
private final TopicPublishInfo topicPublishInfo;
private final long timeout;
private volatile LogicalQueuesInfo logicalQueuesInfo;
private volatile LogicalQueueRouteData writableQueueRouteData;
private final AtomicInteger retry = new AtomicInteger();
public LogicalQueueSendContext(Message msg, MessageQueue mq,
CommunicationMode communicationMode, SendCallback sendCallback,
TopicPublishInfo topicPublishInfo, long timeout) {
this.msg = msg;
this.mq = mq;
this.communicationMode = communicationMode;
this.sendCallback = sendCallback;
this.topicPublishInfo = topicPublishInfo;
this.timeout = timeout;
if (topicPublishInfo == null) {
topicPublishInfo = DefaultMQProducerImpl.this.tryToFindTopicPublishInfo(mq.getTopic());
}
if (topicPublishInfo != null) {
this.logicalQueuesInfo = topicPublishInfo.getTopicRouteData().getLogicalQueuesInfo();
} else {
this.logicalQueuesInfo = null;
}
}
private boolean notUsingLogicalQueue() {
return !Objects.equal(mq.getBrokerName(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME) || this.logicalQueuesInfo == null;
}
public MessageQueue getModifiedMessageQueue() throws MQClientException {
if (this.notUsingLogicalQueue()) {
return this.mq;
}
this.writableQueueRouteData = getWritableQueueRouteData();
MessageQueue mq = new MessageQueue(this.mq);
mq.setBrokerName(writableQueueRouteData.getBrokerName());
mq.setQueueId(writableQueueRouteData.getQueueId());
return mq;
}
private LogicalQueueRouteData getWritableQueueRouteData() throws MQClientException {
this.logicalQueuesInfo.readLock().lock();
try {
List<LogicalQueueRouteData> queueRouteDataList = logicalQueuesInfo.get(mq.getQueueId());
if (queueRouteDataList == null || queueRouteDataList.size() == 0) {
throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no queue route data found", mq.getQueueId()), null);
}
// usually writable queue is placed in the last position, or second last when queue migrating
for (int i = queueRouteDataList.size() - 1; i >= 0; i--) {
LogicalQueueRouteData queueRouteData = queueRouteDataList.get(i);
if (queueRouteData.isWritable()) {
return queueRouteData;
}
}
throw new MQClientException(String.format(Locale.ENGLISH, "send to a logical queue %d but no writable queue route data found", mq.getQueueId()), null);
} finally {
this.logicalQueuesInfo.readLock().unlock();
}
}
@Override public void onSuccess(SendResult sendResult) {
this.sendCallback.onSuccess(this.wrapSendResult(sendResult));
}
@Override public void onException(Throwable t) {
if (this.shouldRetry(t)) {
try {
DefaultMQProducerImpl.this.sendKernelImplWithoutRetry(msg, this.getModifiedMessageQueue(), communicationMode, this, topicPublishInfo, timeout);
return;
} catch (Exception e) {
t = e;
}
}
if (t instanceof MQRedirectException) {
t = new MQBrokerException(ResponseCode.SYSTEM_ERROR, "redirect");
}
this.sendCallback.onException(t);
}
private void handleRedirectException(MQRedirectException re) {
byte[] responseBody = re.getBody();
log.info("LogicalQueueContext.processResponseBody got redirect {}: {}", this.writableQueueRouteData, responseBody != null ? new String(responseBody, MessageDecoder.CHARSET_UTF8) : null);
try {
List<LogicalQueueRouteData> newQueueRouteDataList = JSON.parseObject(responseBody, MixAll.TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA);
this.logicalQueuesInfo.updateLogicalQueueRouteDataList(this.mq.getQueueId(), newQueueRouteDataList);
} catch (Exception e) {
log.warn("LogicalQueueContext.processResponseBody {} update exception, fallback to updateTopicRouteInfoFromNameServer", this.writableQueueRouteData, e);
DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId()));
TopicRouteData topicRouteData = DefaultMQProducerImpl.this.mQClientFactory.getAnExistTopicRouteData(mq.getTopic());
if (topicRouteData != null) {
this.logicalQueuesInfo = topicRouteData.getLogicalQueuesInfo();
} else {
this.logicalQueuesInfo = null;
}
}
}
public SendCallback wrapSendCallback() {
if (this.notUsingLogicalQueue()) {
return this.sendCallback;
}
if (!CommunicationMode.ASYNC.equals(this.communicationMode)) {
return this.sendCallback;
}
return this;
}
public boolean shouldRetry(Throwable t) {
this.incrRetry();
if (this.exceedMaxRetry()) {
log.warn("retry {} too many times: {}", this.retry.get(), this.writableQueueRouteData);
return false;
}
if (!this.writableQueueRouteData.isWritable()) {
log.warn("no writable queue: {}", this.writableQueueRouteData);
return false;
}
if (t instanceof MQRedirectException) {
this.handleRedirectException((MQRedirectException) t);
return true;
}
return !(t instanceof RemotingException) || this.handleRemotingException((RemotingException) t);
}
public boolean exceedMaxRetry() {
return this.retry.get() >= 3;
}
public void incrRetry() {
this.retry.incrementAndGet();
}
public SendResult wrapSendResult(SendResult sendResult) {
if (sendResult == null) {
return null;
}
SendResultForLogicalQueue newSendResult = new SendResultForLogicalQueue(sendResult, this.writableQueueRouteData.getLogicalQueueIndex());
long queueOffset = newSendResult.getQueueOffset();
if (queueOffset >= 0) {
newSendResult.setQueueOffset(LogicalQueueSendContext.this.writableQueueRouteData.toLogicalQueueOffset(queueOffset));
}
return newSendResult;
}
public boolean handleRemotingException(RemotingException e) {
if (e instanceof RemotingTooMuchRequestException) {
return false;
}
DefaultMQProducerImpl.this.mQClientFactory.updateTopicRouteInfoFromNameServer(this.mq.getTopic(), false, null, Collections.singleton(mq.getQueueId()));
this.logicalQueuesInfo = DefaultMQProducerImpl.this.getTopicPublishInfoTable().get(mq.getTopic()).getTopicRouteData().getLogicalQueuesInfo();
LogicalQueueRouteData writableQueueRouteData;
try {
writableQueueRouteData = this.getWritableQueueRouteData();
} catch (MQClientException ce) {
log.warn("getWritableQueueRouteData exception: {}", this.logicalQueuesInfo.get(mq.getQueueId()), ce);
return false;
}
if (Objects.equal(this.writableQueueRouteData.getMessageQueue(), writableQueueRouteData.getMessageQueue()) && writableQueueRouteData.isWritable()) {
// still same MessageQueue and still writable, no need to retry
return false;
}
return true;
}
}
}
client/src/main/java/org/apache/rocketmq/client/producer/SendResultForLogicalQueue.java
-46
View File
@@ -1,46 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.client.producer;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageQueue;
public class SendResultForLogicalQueue extends SendResult {
private final String origBrokerName;
private final int origQueueId;
public SendResultForLogicalQueue(SendResult sendResult, int logicalQueueIdx) {
super(sendResult.getSendStatus(), sendResult.getMsgId(), sendResult.getOffsetMsgId(), new MessageQueue(sendResult.getMessageQueue().getTopic(), MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, logicalQueueIdx), sendResult.getQueueOffset());
this.origBrokerName = sendResult.getMessageQueue().getBrokerName();
this.origQueueId = sendResult.getMessageQueue().getQueueId();
}
public String getOrigBrokerName() {
return origBrokerName;
}
public int getOrigQueueId() {
return origQueueId;
}
@Override public String toString() {
return "SendResultForLogicalQueue{" +
"origBrokerName='" + origBrokerName + '\'' +
", origQueueId=" + origQueueId +
"} " + super.toString();
}
}
client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumerLogicalQueueTest.java
-248
View File
@@ -1,248 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.client.consumer;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.SettableFuture;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.impl.CommunicationMode;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.MQClientAPIImpl;
import org.apache.rocketmq.client.impl.MQClientManager;
import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper;
import org.apache.rocketmq.client.impl.consumer.PullResultExt;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class DefaultMQPullConsumerLogicalQueueTest {
private MQClientInstance mQClientFactory;
@Mock
private MQClientAPIImpl mQClientAPIImpl;
private DefaultMQPullConsumer pullConsumer;
private String topic;
private static final String cluster = "DefaultCluster";
private static final String broker1Name = "BrokerA";
private static final String broker1Addr = "127.0.0.2:10911";
private static final String broker2Name = "BrokerB";
private static final String broker2Addr = "127.0.0.3:10911";
@Before
public void init() throws Exception {
topic = "FooBar" + System.nanoTime();
mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()));
FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true);
pullConsumer = new DefaultMQPullConsumer("FooBarGroup" + System.nanoTime());
pullConsumer.setNamesrvAddr("127.0.0.1:9876");
pullConsumer.start();
PullAPIWrapper pullAPIWrapper = pullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper();
FieldUtils.writeDeclaredField(pullAPIWrapper, "mQClientFactory", mQClientFactory, true);
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRouteData());
doReturn(new FindBrokerResult(broker1Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker1Name), anyLong(), anyBoolean());
doReturn(new FindBrokerResult(broker2Addr, false)).when(mQClientFactory).findBrokerAddressInSubscribe(eq(broker2Name), anyLong(), anyBoolean());
}
@After
public void terminate() {
pullConsumer.shutdown();
}
@Test
public void testStart_OffsetShouldNotNUllAfterStart() {
Assert.assertNotNull(pullConsumer.getOffsetStore());
}
@Test
public void testPullMessage_Success() throws Exception {
doAnswer(new Answer<PullResultExt>() {
@Override public PullResultExt answer(InvocationOnMock mock) throws Throwable {
PullMessageRequestHeader requestHeader = mock.getArgument(1);
return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt()));
}
}).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull());
MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0);
PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3);
assertThat(pullResult).isNotNull();
assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND);
assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1);
assertThat(pullResult.getMinOffset()).isEqualTo(123);
assertThat(pullResult.getMaxOffset()).isEqualTo(2048);
assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList());
}
@Test
public void testPullMessage_NotFound() throws Exception {
doAnswer(new Answer<PullResult>() {
@Override public PullResult answer(InvocationOnMock mock) throws Throwable {
PullMessageRequestHeader requestHeader = mock.getArgument(1);
return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.NO_NEW_MSG, new ArrayList<MessageExt>());
}
}).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull());
MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0);
PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3);
assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG);
}
@Test
public void testPullMessageAsync_Success() throws Exception {
doAnswer(new Answer<PullResult>() {
@Override public PullResult answer(InvocationOnMock mock) throws Throwable {
PullMessageRequestHeader requestHeader = mock.getArgument(1);
PullResult pullResult = DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt()));
PullCallback pullCallback = mock.getArgument(4);
pullCallback.onSuccess(pullResult);
return null;
}
}).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.ASYNC), any(PullCallback.class));
final SettableFuture<PullResult> future = SettableFuture.create();
MessageQueue messageQueue = new MessageQueue(topic, broker1Name, 0);
pullConsumer.pull(messageQueue, "*", 1024, 3, new PullCallback() {
@Override
public void onSuccess(PullResult pullResult) {
future.set(pullResult);
}
@Override
public void onException(Throwable e) {
future.setException(e);
}
});
PullResult pullResult = future.get(3, TimeUnit.SECONDS);
assertThat(pullResult).isNotNull();
assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND);
assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1);
assertThat(pullResult.getMinOffset()).isEqualTo(123);
assertThat(pullResult.getMaxOffset()).isEqualTo(2048);
assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList());
}
@Test
public void testPullMessageSync_Redirect() throws Exception {
doAnswer(new Answer<PullResult>() {
@Override public PullResult answer(InvocationOnMock mock) throws Throwable {
throw new MQRedirectException(JSON.toJSONBytes(ImmutableList.of(
new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr),
new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr)
)));
}
}).when(mQClientAPIImpl).pullMessage(eq(broker1Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull());
doAnswer(new Answer<PullResult>() {
@Override public PullResult answer(InvocationOnMock mock) throws Throwable {
PullMessageRequestHeader requestHeader = mock.getArgument(1);
return DefaultMQPullConsumerLogicalQueueTest.this.createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(new MessageExt()));
}
}).when(mQClientAPIImpl).pullMessage(eq(broker2Addr), any(PullMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC), (PullCallback) isNull());
MessageQueue messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0);
PullResult pullResult = pullConsumer.pull(messageQueue, "*", 1024, 3);
assertThat(pullResult).isNotNull();
assertThat(pullResult.getPullStatus()).isEqualTo(PullStatus.FOUND);
assertThat(pullResult.getNextBeginOffset()).isEqualTo(1024 + 1);
assertThat(pullResult.getMinOffset()).isEqualTo(123 + 10);
assertThat(pullResult.getMaxOffset()).isEqualTo(2048 + 10);
assertThat(pullResult.getMsgFoundList()).isEqualTo(Collections.emptyList());
}
private TopicRouteData createTopicRouteData() {
TopicRouteData topicRouteData = new TopicRouteData();
topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
topicRouteData.setBrokerDatas(ImmutableList.of(
new BrokerData(cluster, broker1Name, new HashMap<Long, String>(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))),
new BrokerData(cluster, broker2Name, new HashMap<Long, String>(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr)))
));
List<QueueData> queueDataList = new ArrayList<QueueData>();
QueueData queueData;
queueData = new QueueData();
queueData.setBrokerName(broker1Name);
queueData.setPerm(6);
queueData.setReadQueueNums(3);
queueData.setWriteQueueNums(4);
queueData.setTopicSysFlag(0);
queueDataList.add(queueData);
queueData = new QueueData();
queueData.setBrokerName(broker2Name);
queueData.setPerm(6);
queueData.setReadQueueNums(3);
queueData.setWriteQueueNums(4);
queueData.setTopicSysFlag(0);
queueDataList.add(queueData);
topicRouteData.setQueueDatas(queueDataList);
LogicalQueuesInfo info = new LogicalQueuesInfo();
info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr)));
topicRouteData.setLogicalQueuesInfo(info);
return topicRouteData;
}
private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus,
List<MessageExt> messageExtList) throws Exception {
return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, new byte[] {});
}
}
client/src/test/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStoreTest.java
+9 -1
View File
@@ -20,10 +20,12 @@ import java.util.Collections;
import java.util.HashSet;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.OffsetNotFoundException;
import org.apache.rocketmq.client.impl.FindBrokerResult;
import org.apache.rocketmq.client.impl.MQClientAPIImpl;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader;
import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
import org.apache.rocketmq.remoting.exception.RemotingException;
@@ -60,6 +62,7 @@ public class RemoteBrokerOffsetStoreTest {
when(mQClientFactory.getClientId()).thenReturn(clientId);
when(mQClientFactory.findBrokerAddressInAdmin(brokerName)).thenReturn(new FindBrokerResult("127.0.0.1", false));
when(mQClientFactory.getMQClientAPIImpl()).thenReturn(mqClientAPI);
when(mQClientFactory.getBrokerNameFromMessageQueue(any())).thenReturn(brokerName);
}
@Test
@@ -84,10 +87,15 @@ public class RemoteBrokerOffsetStoreTest {
offsetStore.updateOffset(messageQueue, 1024, false);
doThrow(new MQBrokerException(-1, "", null))
doThrow(new OffsetNotFoundException(ResponseCode.PULL_NOT_FOUND, "", null))
.when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());
assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-1);
doThrow(new MQBrokerException(-1, "", null))
.when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());
assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2);
doThrow(new RemotingException("", null))
.when(mqClientAPI).queryConsumerOffset(anyString(), any(QueryConsumerOffsetRequestHeader.class), anyLong());
assertThat(offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE)).isEqualTo(-2);
client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerLogicalQueueTest.java
-311
View File
@@ -1,311 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.client.producer;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.SettableFuture;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.exception.MQRedirectException;
import org.apache.rocketmq.client.hook.SendMessageContext;
import org.apache.rocketmq.client.impl.CommunicationMode;
import org.apache.rocketmq.client.impl.MQClientAPIImpl;
import org.apache.rocketmq.client.impl.MQClientManager;
import org.apache.rocketmq.client.impl.factory.MQClientInstance;
import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl;
import org.apache.rocketmq.client.impl.producer.TopicPublishInfo;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.Message;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.exception.RemotingConnectException;
import org.assertj.core.api.ThrowableAssert;
import org.assertj.core.util.Lists;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class DefaultMQProducerLogicalQueueTest {
private MQClientInstance mQClientFactory;
@Mock
private MQClientAPIImpl mQClientAPIImpl;
private DefaultMQProducer producer;
private Message message;
private String topic;
private MessageQueue messageQueue;
private static final String cluster = "DefaultCluster";
private static final String broker1Name = "broker1";
private static final String broker2Name = "broker2";
private static final String broker1Addr = "127.0.0.2:10911";
private static final String broker2Addr = "127.0.0.3:10911";
@Before
public void init() throws Exception {
topic = "Foobar" + System.nanoTime();
messageQueue = new MessageQueue(topic, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME, 0);
ConcurrentMap<String, MQClientInstance> factoryTable = (ConcurrentMap<String/* clientId */, MQClientInstance>) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true);
for (MQClientInstance instance : factoryTable.values()) {
instance.shutdown();
}
factoryTable.clear();
mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()));
factoryTable.put(new ClientConfig().buildMQClientId(), mQClientFactory);
String producerGroupTemp = "FooBar_PID" + System.nanoTime();
producer = new DefaultMQProducer(producerGroupTemp);
producer.setNamesrvAddr("127.0.0.1:9876");
producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE);
message = new Message(topic, new byte[] {'a'});
mQClientFactory.registerProducer(producerGroupTemp, producer.getDefaultMQProducerImpl());
producer.start();
FieldUtils.writeDeclaredField(producer.getDefaultMQProducerImpl(), "mQClientFactory", mQClientFactory, true);
FieldUtils.writeField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true);
when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),
nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod();
when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenReturn(createSendResult(SendStatus.SEND_OK));
when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class),
any(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenAnswer(new Answer<SendResult>() {
@Override public SendResult answer(InvocationOnMock invocation) throws Throwable {
SendCallback sendCallback = invocation.getArgument(6);
sendCallback.onSuccess(DefaultMQProducerLogicalQueueTest.this.createSendResult(SendStatus.SEND_OK));
return null;
}
});
}
@After
public void terminate() {
producer.shutdown();
}
@Test
public void testSendMessageSync_Success() throws Exception {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
SendResult sendResult = producer.send(message, messageQueue);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
assertThat(sendResult.getOffsetMsgId()).isEqualTo("123");
assertThat(sendResult.getQueueOffset()).isEqualTo(456L);
}
@Test
public void testSendMessageSync_Redirect() throws Exception {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenThrow(new MQRedirectException(null));
assertThatThrownBy(new ThrowableAssert.ThrowingCallable() {
@Override public void call() throws Throwable {
producer.send(message, messageQueue);
}
}).isInstanceOf(MQBrokerException.class).hasMessageContaining("redirect");
when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenThrow(new MQRedirectException(JSON.toJSONBytes(ImmutableList.of(
new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Expired, 0, 0, 0, 0, broker1Addr),
new LogicalQueueRouteData(0, 10, new MessageQueue(topic, broker2Name, 0), MessageQueueRouteState.Normal, 0, -1, -1, -1, broker2Addr)))));
when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenReturn(createSendResult(SendStatus.SEND_OK));
SendResult sendResult = producer.send(message, messageQueue);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
assertThat(sendResult.getOffsetMsgId()).isEqualTo("123");
assertThat(sendResult.getQueueOffset()).isEqualTo(466L);
}
@Test
public void testSendMessageSync_RemotingException() throws Exception {
TopicRouteData topicRouteData = createTopicRoute();
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(topicRouteData);
when(mQClientAPIImpl.sendMessage(eq(broker1Addr), eq(broker1Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenThrow(new RemotingConnectException(broker1Addr));
SendResult returnSendResult = createSendResult(SendStatus.SEND_OK);
when(mQClientAPIImpl.sendMessage(eq(broker2Addr), eq(broker2Name), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), eq(CommunicationMode.SYNC),
(SendCallback) isNull(), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class)))
.thenReturn(returnSendResult);
assertThatThrownBy(new ThrowableAssert.ThrowingCallable() {
@Override public void call() throws Throwable {
producer.send(message, messageQueue);
}
}).isInstanceOf(RemotingConnectException.class).hasMessageContaining(broker1Addr);
topicRouteData.getLogicalQueuesInfo().get(0).add(new LogicalQueueRouteData(0, -1, new MessageQueue(topic, broker2Name, 1), MessageQueueRouteState.WriteOnly, 0, -1, -1, -1, broker2Addr));
SendResult sendResult = producer.send(message, messageQueue);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
assertThat(sendResult.getOffsetMsgId()).isEqualTo("123");
assertThat(sendResult.getQueueOffset()).isEqualTo(-1L);
}
@Test
public void testSendMessageAsync_Success() throws Exception {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
final SettableFuture<SendResult> future = SettableFuture.create();
producer.send(message, messageQueue, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
future.set(sendResult);
}
@Override
public void onException(Throwable e) {
future.setException(e);
}
});
SendResult sendResult = future.get(3, TimeUnit.SECONDS);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
assertThat(sendResult.getOffsetMsgId()).isEqualTo("123");
assertThat(sendResult.getQueueOffset()).isEqualTo(456L);
}
@Test
public void testSendMessageAsync() throws Exception {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
final AtomicReference<SettableFuture<SendResult>> future = new AtomicReference<SettableFuture<SendResult>>();
SendCallback sendCallback = new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
future.get().set(sendResult);
}
@Override
public void onException(Throwable e) {
future.get().setException(e);
}
};
Message message = new Message();
message.setTopic("test");
message.setBody("hello world".getBytes());
future.set(SettableFuture.<SendResult>create());
producer.send(new Message(), messageQueue, sendCallback);
assertThatThrownBy(new ThrowableAssert.ThrowingCallable() {
@Override public void call() throws Throwable {
future.get().get(3, TimeUnit.SECONDS);
}
}).hasCauseInstanceOf(MQClientException.class).hasMessageContaining("The specified topic is blank");
//this message is send success
message.setTopic(topic);
future.set(SettableFuture.<SendResult>create());
producer.send(message, messageQueue, sendCallback, 1000);
future.get().get(3, TimeUnit.SECONDS);
}
public TopicRouteData createTopicRoute() {
TopicRouteData topicRouteData = new TopicRouteData();
topicRouteData.setFilterServerTable(new HashMap<String, List<String>>());
topicRouteData.setBrokerDatas(ImmutableList.of(
new BrokerData(cluster, broker1Name, new HashMap<Long, String>(Collections.singletonMap(MixAll.MASTER_ID, broker1Addr))),
new BrokerData(cluster, broker2Name, new HashMap<Long, String>(Collections.singletonMap(MixAll.MASTER_ID, broker2Addr)))
));
List<QueueData> queueDataList = new ArrayList<QueueData>();
QueueData queueData;
queueData = new QueueData();
queueData.setBrokerName(broker1Name);
queueData.setPerm(6);
queueData.setReadQueueNums(3);
queueData.setWriteQueueNums(4);
queueData.setTopicSysFlag(0);
queueDataList.add(queueData);
queueData = new QueueData();
queueData.setBrokerName(broker2Name);
queueData.setPerm(6);
queueData.setReadQueueNums(3);
queueData.setWriteQueueNums(4);
queueData.setTopicSysFlag(0);
queueDataList.add(queueData);
topicRouteData.setQueueDatas(queueDataList);
LogicalQueuesInfo info = new LogicalQueuesInfo();
info.put(0, Lists.newArrayList(new LogicalQueueRouteData(0, 0, new MessageQueue(topic, broker1Name, 0), MessageQueueRouteState.Normal, 0, 0, 0, 0, broker1Addr)));
topicRouteData.setLogicalQueuesInfo(info);
return topicRouteData;
}
private SendResult createSendResult(SendStatus sendStatus) {
SendResult sendResult = new SendResult();
sendResult.setMsgId("123");
sendResult.setOffsetMsgId("123");
sendResult.setQueueOffset(456);
sendResult.setSendStatus(sendStatus);
sendResult.setRegionId("HZ");
sendResult.setMessageQueue(new MessageQueue(topic, broker1Name, 0));
return sendResult;
}
}
client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java
+14 -14
View File
@@ -22,7 +22,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
@@ -49,20 +48,21 @@ import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.netty.NettyRemotingClient;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
@@ -153,7 +153,7 @@ public class DefaultMQProducerTest {
@Test
public void testSendMessageSync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
SendResult sendResult = producer.send(message);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
@@ -163,7 +163,7 @@ public class DefaultMQProducerTest {
@Test
public void testSendMessageSync_WithBodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
SendResult sendResult = producer.send(bigMessage);
assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK);
@@ -174,7 +174,7 @@ public class DefaultMQProducerTest {
@Test
public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
final CountDownLatch countDownLatch = new CountDownLatch(1);
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
producer.send(message, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
@@ -197,7 +197,7 @@ public class DefaultMQProducerTest {
final AtomicInteger cc = new AtomicInteger(0);
final CountDownLatch countDownLatch = new CountDownLatch(6);
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
SendCallback sendCallback = new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
@@ -239,7 +239,7 @@ public class DefaultMQProducerTest {
final AtomicInteger cc = new AtomicInteger(0);
final CountDownLatch countDownLatch = new CountDownLatch(4);
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
SendCallback sendCallback = new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
@@ -260,7 +260,7 @@ public class DefaultMQProducerTest {
}
};
List<Message> msgs = new ArrayList<Message>();
List<Message> msgs = new ArrayList<>();
for (int i = 0; i < 5; i++) {
Message message = new Message();
message.setTopic("test");
@@ -281,7 +281,7 @@ public class DefaultMQProducerTest {
@Test
public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
final CountDownLatch countDownLatch = new CountDownLatch(1);
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
producer.send(bigMessage, new SendCallback() {
@Override
public void onSuccess(SendResult sendResult) {
@@ -300,7 +300,7 @@ public class DefaultMQProducerTest {
@Test
public void testSendMessageSync_SuccessWithHook() throws Throwable {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
final Throwable[] assertionErrors = new Throwable[1];
final CountDownLatch countDownLatch = new CountDownLatch(2);
producer.getDefaultMQProducerImpl().registerSendMessageHook(new SendMessageHook() {
@@ -368,7 +368,7 @@ public class DefaultMQProducerTest {
@Test
public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
final AtomicBoolean finish = new AtomicBoolean(false);
new Thread(new Runnable() {
@Override public void run() {
@@ -394,13 +394,13 @@ public class DefaultMQProducerTest {
@Test(expected = RequestTimeoutException.class)
public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
Message result = producer.request(message, 3 * 1000L);
}
@Test
public void testAsyncRequest_OnSuccess() throws Exception {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
final CountDownLatch countDownLatch = new CountDownLatch(1);
RequestCallback requestCallback = new RequestCallback() {
@Override public void onSuccess(Message message) {
client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithOpenTracingTest.java
+6 -8
View File
@@ -20,11 +20,6 @@ package org.apache.rocketmq.client.trace;
import io.opentracing.mock.MockSpan;
import io.opentracing.mock.MockTracer;
import io.opentracing.tag.Tags;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -52,14 +47,17 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
@@ -115,7 +113,7 @@ public class DefaultMQProducerWithOpenTracingTest {
@Test
public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
producer.send(message);
assertThat(tracer.finishedSpans().size()).isEqualTo(1);
MockSpan span = tracer.finishedSpans().get(0);
client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQProducerWithTraceTest.java
+10 -16
View File
@@ -17,13 +17,6 @@
package org.apache.rocketmq.client.trace;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -50,17 +43,18 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
@@ -128,7 +122,7 @@ public class DefaultMQProducerWithTraceTest {
@Test
public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
final CountDownLatch countDownLatch = new CountDownLatch(1);
try {
producer.send(message);
@@ -140,7 +134,7 @@ public class DefaultMQProducerWithTraceTest {
@Test
public void testSendMessageSync_WithTrace_NoBrokerSet_Exception() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
final CountDownLatch countDownLatch = new CountDownLatch(1);
try {
producer.send(message);
client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithOpenTracingTest.java
+7 -9
View File
@@ -20,12 +20,6 @@ package org.apache.rocketmq.client.trace;
import io.opentracing.mock.MockSpan;
import io.opentracing.mock.MockTracer;
import io.opentracing.tag.Tags;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -59,14 +53,18 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
@@ -133,7 +131,7 @@ public class TransactionMQProducerWithOpenTracingTest {
@Test
public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
producer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, producer.getDefaultMQProducerImpl());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
producer.sendMessageInTransaction(message, null);
assertThat(tracer.finishedSpans().size()).isEqualTo(2);
client/src/test/java/org/apache/rocketmq/client/trace/TransactionMQProducerWithTraceTest.java
+15 -25
View File
@@ -17,13 +17,6 @@
package org.apache.rocketmq.client.trace;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -57,20 +50,19 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentMatchers;
import org.mockito.Mock;
import org.mockito.Spy;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.junit.MockitoJUnitRunner;
import org.mockito.stubbing.Answer;
import java.lang.reflect.Field;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.nullable;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.when;
@@ -135,7 +127,7 @@ public class TransactionMQProducerWithTraceTest {
Field fieldHooks = DefaultMQProducerImpl.class.getDeclaredField("endTransactionHookList");
fieldHooks.setAccessible(true);
List<EndTransactionHook>hooks = new ArrayList<EndTransactionHook>();
List<EndTransactionHook>hooks = new ArrayList<>();
hooks.add(endTransactionHook);
fieldHooks.set(producer.getDefaultMQProducerImpl(), hooks);
@@ -150,14 +142,12 @@ public class TransactionMQProducerWithTraceTest {
@Test
public void testSendMessageSync_WithTrace_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException {
traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong(), anyBoolean(), ArgumentMatchers.<Set<Integer>>any())).thenReturn(createTopicRoute());
final AtomicReference<EndTransactionContext> context = new AtomicReference<EndTransactionContext>();
doAnswer(new Answer() {
@Override public Object answer(InvocationOnMock mock) throws Throwable {
context.set(mock.<EndTransactionContext>getArgument(0));
return null;
}
}).when(endTransactionHook).endTransaction(ArgumentMatchers.<EndTransactionContext>any());
when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute());
AtomicReference<EndTransactionContext> context = new AtomicReference<>();
doAnswer(mock -> {
context.set(mock.getArgument(0));
return null;
}).when(endTransactionHook).endTransaction(any());
producer.sendMessageInTransaction(message, null);
EndTransactionContext ctx = context.get();
common/src/main/java/org/apache/rocketmq/common/MixAll.java
+9 -11
View File
@@ -16,7 +16,12 @@
*/
package org.apache.rocketmq.common;
import com.alibaba.fastjson.TypeReference;
import org.apache.rocketmq.common.annotation.ImportantField;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -27,7 +32,6 @@ import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.NetworkInterface;
@@ -40,12 +44,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.common.annotation.ImportantField;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
public class MixAll {
public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME";
@@ -87,9 +85,9 @@ public class MixAll {
public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml";
public static final String REPLY_MESSAGE_FLAG = "reply";
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME = "__logical_queue_broker__";
public static final Type TYPE_LIST_LOGICAL_QUEUE_ROUTE_DATA = new TypeReference<List<LogicalQueueRouteData>>() {
}.getType();
public static final String LOGICAL_QUEUE_MOCK_BROKER_PREFIX = "__syslo__";
public static final String METADATA_SCOPE_GLOBAL = "__global__";
public static final String LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST = "__syslo__none__";
public static String getWSAddr() {
String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP);
common/src/main/java/org/apache/rocketmq/common/TopicConfig.java
+6
View File
@@ -47,6 +47,12 @@ public class TopicConfig {
this.order = other.order;
}
public TopicConfig(String topicName, int readQueueNums, int writeQueueNums) {
this.topicName = topicName;
this.readQueueNums = readQueueNums;
this.writeQueueNums = writeQueueNums;
}
public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) {
this.topicName = topicName;
this.readQueueNums = readQueueNums;
common/src/main/java/org/apache/rocketmq/common/admin/TopicOffset.java
+9
View File
@@ -44,4 +44,13 @@ public class TopicOffset {
public void setLastUpdateTimestamp(long lastUpdateTimestamp) {
this.lastUpdateTimestamp = lastUpdateTimestamp;
}
@Override
public String toString() {
return "TopicOffset{" +
"minOffset=" + minOffset +
", maxOffset=" + maxOffset +
", lastUpdateTimestamp=" + lastUpdateTimestamp +
'}';
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java
+2 -9
View File
@@ -200,13 +200,6 @@ public class RequestCode {
public static final int QUERY_ASSIGNMENT = 400;
public static final int SET_MESSAGE_REQUEST_MODE = 401;
public static final int UPDATE_TOPIC_LOGICAL_QUEUE_MAPPING = 411;
public static final int DELETE_TOPIC_LOGICAL_QUEUE_MAPPING = 422;
public static final int QUERY_TOPIC_LOGICAL_QUEUE_MAPPING = 413;
public static final int SEAL_TOPIC_LOGICAL_QUEUE = 414;
public static final int REUSE_TOPIC_LOGICAL_QUEUE = 415;
public static final int CREATE_MESSAGE_QUEUE_FOR_LOGICAL_QUEUE = 416;
public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_PREPARE = 417;
public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_COMMIT = 418;
public static final int MIGRATE_TOPIC_LOGICAL_QUEUE_NOTIFY = 419;
public static final int UPDATE_AND_CREATE_STATIC_TOPIC = 513;
}
common/src/main/java/org/apache/rocketmq/common/protocol/ResponseCode.java
+9
View File
@@ -84,4 +84,13 @@ public class ResponseCode extends RemotingSysResponseCode {
public static final int POLLING_FULL = 209;
public static final int POLLING_TIMEOUT = 210;
public static final int NOT_LEADER_FOR_QUEUE = 501;
public static final int RPC_UNKNOWN = -1000;
public static final int RPC_ADDR_IS_NULL = -1002;
public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004;
public static final int RPC_TIME_OUT = -1006;
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/CreateMessageQueueForLogicalQueueRequestBody.java
-50
View File
@@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class CreateMessageQueueForLogicalQueueRequestBody extends RemotingSerializable {
private String topic;
private int logicalQueueIndex;
private MessageQueueRouteState messageQueueStatus;
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public int getLogicalQueueIndex() {
return logicalQueueIndex;
}
public void setLogicalQueueIndex(int logicalQueueIndex) {
this.logicalQueueIndex = logicalQueueIndex;
}
public MessageQueueRouteState getMessageQueueStatus() {
return messageQueueStatus;
}
public void setMessageQueueStatus(MessageQueueRouteState messageQueueStatuses) {
this.messageQueueStatus = messageQueueStatuses;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/MigrateLogicalQueueBody.java
-42
View File
@@ -1,42 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
import org.apache.rocketmq.common.protocol.route.LogicalQueueRouteData;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class MigrateLogicalQueueBody extends RemotingSerializable {
private LogicalQueueRouteData fromQueueRouteData;
private LogicalQueueRouteData toQueueRouteData;
public LogicalQueueRouteData getFromQueueRouteData() {
return fromQueueRouteData;
}
public void setFromQueueRouteData(
LogicalQueueRouteData fromQueueRouteData) {
this.fromQueueRouteData = fromQueueRouteData;
}
public LogicalQueueRouteData getToQueueRouteData() {
return toQueueRouteData;
}
public void setToQueueRouteData(LogicalQueueRouteData toQueueRouteData) {
this.toQueueRouteData = toQueueRouteData;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/RegisterBrokerBody.java
+29 -3
View File
@@ -33,6 +33,7 @@ import java.util.zip.InflaterInputStream;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
@@ -41,7 +42,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class RegisterBrokerBody extends RemotingSerializable {
private static final InternalLogger LOGGER = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper();
private TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();
private List<String> filterServerList = new ArrayList<String>();
public byte[] encode(boolean compress) {
@@ -82,6 +83,20 @@ public class RegisterBrokerBody extends RemotingSerializable {
// write filter server list json
outputStream.write(buffer);
//write the topic queue mapping
Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = topicConfigSerializeWrapper.getTopicQueueMappingInfoMap();
if (topicQueueMappingInfoMap == null) {
//as the place holder
topicQueueMappingInfoMap = new ConcurrentHashMap<String, TopicQueueMappingInfo>();
}
outputStream.write(convertIntToByteArray(topicQueueMappingInfoMap.size()));
for (TopicQueueMappingInfo info: topicQueueMappingInfoMap.values()) {
buffer = JSON.toJSONString(info).getBytes(MixAll.DEFAULT_CHARSET);
outputStream.write(convertIntToByteArray(buffer.length));
// write filter server list json
outputStream.write(buffer);
}
outputStream.finish();
long interval = System.currentTimeMillis() - start;
if (interval > 50) {
@@ -134,6 +149,17 @@ public class RegisterBrokerBody extends RemotingSerializable {
}
registerBrokerBody.setFilterServerList(filterServerList);
int topicQueueMappingNum = readInt(inflaterInputStream);
Map<String/* topic */, TopicQueueMappingInfo> topicQueueMappingInfoMap = new ConcurrentHashMap<String, TopicQueueMappingInfo>();
for (int i = 0; i < topicQueueMappingNum; i++) {
int mappingJsonLen = readInt(inflaterInputStream);
byte[] buffer = readBytes(inflaterInputStream, mappingJsonLen);
TopicQueueMappingInfo info = TopicQueueMappingInfo.decode(buffer, TopicQueueMappingInfo.class);
topicQueueMappingInfoMap.put(info.getTopic(), info);
}
registerBrokerBody.getTopicConfigSerializeWrapper().setTopicQueueMappingInfoMap(topicQueueMappingInfoMap);
long interval = System.currentTimeMillis() - start;
if (interval > 50) {
LOGGER.info("Decompressing takes {}ms", interval);
@@ -167,11 +193,11 @@ public class RegisterBrokerBody extends RemotingSerializable {
return byteBuffer.getInt();
}
public TopicConfigSerializeWrapper getTopicConfigSerializeWrapper() {
public TopicConfigAndMappingSerializeWrapper getTopicConfigSerializeWrapper() {
return topicConfigSerializeWrapper;
}
public void setTopicConfigSerializeWrapper(TopicConfigSerializeWrapper topicConfigSerializeWrapper) {
public void setTopicConfigSerializeWrapper(TopicConfigAndMappingSerializeWrapper topicConfigSerializeWrapper) {
this.topicConfigSerializeWrapper = topicConfigSerializeWrapper;
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/ReuseTopicLogicalQueueRequestBody.java
-59
View File
@@ -1,59 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
import org.apache.rocketmq.common.protocol.route.MessageQueueRouteState;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class ReuseTopicLogicalQueueRequestBody extends RemotingSerializable {
private String topic;
private int queueId;
private int logicalQueueIndex;
private MessageQueueRouteState messageQueueRouteState;
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public int getQueueId() {
return queueId;
}
public void setQueueId(int queueId) {
this.queueId = queueId;
}
public int getLogicalQueueIndex() {
return logicalQueueIndex;
}
public void setLogicalQueueIndex(int logicalQueueIndex) {
this.logicalQueueIndex = logicalQueueIndex;
}
public void setMessageQueueRouteState(MessageQueueRouteState messageQueueRouteState) {
this.messageQueueRouteState = messageQueueRouteState;
}
public MessageQueueRouteState getMessageQueueRouteState() {
return messageQueueRouteState;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigAndMappingSerializeWrapper.java
+68
View File
@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class TopicConfigAndMappingSerializeWrapper extends TopicConfigSerializeWrapper {
private Map<String/* topic */, TopicQueueMappingInfo> topicQueueMappingInfoMap = new ConcurrentHashMap<String, TopicQueueMappingInfo>();
private Map<String/* topic */, TopicQueueMappingDetail> topicQueueMappingDetailMap = new ConcurrentHashMap<String, TopicQueueMappingDetail>();
private DataVersion mappingDataVersion = new DataVersion();
public Map<String, TopicQueueMappingInfo> getTopicQueueMappingInfoMap() {
return topicQueueMappingInfoMap;
}
public void setTopicQueueMappingInfoMap(Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap) {
this.topicQueueMappingInfoMap = topicQueueMappingInfoMap;
}
public Map<String, TopicQueueMappingDetail> getTopicQueueMappingDetailMap() {
return topicQueueMappingDetailMap;
}
public void setTopicQueueMappingDetailMap(Map<String, TopicQueueMappingDetail> topicQueueMappingDetailMap) {
this.topicQueueMappingDetailMap = topicQueueMappingDetailMap;
}
public DataVersion getMappingDataVersion() {
return mappingDataVersion;
}
public void setMappingDataVersion(DataVersion mappingDataVersion) {
this.mappingDataVersion = mappingDataVersion;
}
public static TopicConfigAndMappingSerializeWrapper from(TopicConfigSerializeWrapper wrapper) {
if (wrapper instanceof TopicConfigAndMappingSerializeWrapper) {
return (TopicConfigAndMappingSerializeWrapper) wrapper;
}
TopicConfigAndMappingSerializeWrapper mappingSerializeWrapper = new TopicConfigAndMappingSerializeWrapper();
mappingSerializeWrapper.setDataVersion(wrapper.getDataVersion());
mappingSerializeWrapper.setTopicConfigTable(wrapper.getTopicConfigTable());
return mappingSerializeWrapper;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java
-11
View File
@@ -17,18 +17,15 @@
package org.apache.rocketmq.common.protocol.body;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.protocol.route.LogicalQueuesInfo;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class TopicConfigSerializeWrapper extends RemotingSerializable {
private ConcurrentMap<String, TopicConfig> topicConfigTable =
new ConcurrentHashMap<String, TopicConfig>();
private Map<String/* topic */, LogicalQueuesInfo> logicalQueuesInfoMap;
private DataVersion dataVersion = new DataVersion();
public ConcurrentMap<String, TopicConfig> getTopicConfigTable() {
@@ -46,12 +43,4 @@ public class TopicConfigSerializeWrapper extends RemotingSerializable {
public void setDataVersion(DataVersion dataVersion) {
this.dataVersion = dataVersion;
}
public Map<String, LogicalQueuesInfo> getLogicalQueuesInfoMap() {
return logicalQueuesInfoMap;
}
public void setLogicalQueuesInfoMap(Map<String, LogicalQueuesInfo> logicalQueuesInfoMap) {
this.logicalQueuesInfoMap = logicalQueuesInfoMap;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/UpdateTopicLogicalQueueMappingRequestBody.java → common/src/main/java/org/apache/rocketmq/common/protocol/body/TopicQueueMappingSerializeWrapper.java
+16 -20
View File
@@ -14,36 +14,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
import org.apache.rocketmq.common.DataVersion;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingDetail;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class UpdateTopicLogicalQueueMappingRequestBody extends RemotingSerializable {
private String topic;
private int queueId;
private int logicalQueueIdx;
import java.util.Map;
public int getLogicalQueueIdx() {
return logicalQueueIdx;
public class TopicQueueMappingSerializeWrapper extends RemotingSerializable {
private Map<String/* topic */, TopicQueueMappingDetail> topicQueueMappingInfoMap;
private DataVersion dataVersion = new DataVersion();
public Map<String, TopicQueueMappingDetail> getTopicQueueMappingInfoMap() {
return topicQueueMappingInfoMap;
}
public void setLogicalQueueIdx(int logicalQueueIdx) {
this.logicalQueueIdx = logicalQueueIdx;
public void setTopicQueueMappingInfoMap(Map<String, TopicQueueMappingDetail> topicQueueMappingInfoMap) {
this.topicQueueMappingInfoMap = topicQueueMappingInfoMap;
}
public String getTopic() {
return topic;
public DataVersion getDataVersion() {
return dataVersion;
}
public void setTopic(String topic) {
this.topic = topic;
}
public int getQueueId() {
return queueId;
}
public void setQueueId(int queueId) {
this.queueId = queueId;
public void setDataVersion(DataVersion dataVersion) {
this.dataVersion = dataVersion;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java
+12
View File
@@ -23,6 +23,7 @@ package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class CreateTopicRequestHeader implements CommandCustomHeader {
@@ -42,6 +43,9 @@ public class CreateTopicRequestHeader implements CommandCustomHeader {
@CFNotNull
private Boolean order = false;
@CFNullable
private Boolean force = false;
@Override
public void checkFields() throws RemotingCommandException {
try {
@@ -118,4 +122,12 @@ public class CreateTopicRequestHeader implements CommandCustomHeader {
public void setOrder(Boolean order) {
this.order = order;
}
public Boolean getForce() {
return force;
}
public void setForce(Boolean force) {
this.force = force;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java
+6 -2
View File
@@ -20,11 +20,11 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader {
public class GetEarliestMsgStoretimeRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String topic;
@CFNotNull
@@ -34,18 +34,22 @@ public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader
public void checkFields() throws RemotingCommandException {
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java
+6 -19
View File
@@ -20,51 +20,38 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class GetMaxOffsetRequestHeader implements CommandCustomHeader {
public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String topic;
@CFNotNull
private Integer queueId;
private boolean committed;
private boolean logicalQueue;
@Override
public void checkFields() throws RemotingCommandException {
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
public void setCommitted(boolean committed) {
this.committed = committed;
}
public boolean isCommitted() {
return committed;
}
public void setLogicalQueue(boolean logicalQueue) {
this.logicalQueue = logicalQueue;
}
public boolean getLogicalQueue() {
return logicalQueue;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java
+6 -2
View File
@@ -20,11 +20,11 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class GetMinOffsetRequestHeader implements CommandCustomHeader {
public class GetMinOffsetRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String topic;
@CFNotNull
@@ -34,18 +34,22 @@ public class GetMinOffsetRequestHeader implements CommandCustomHeader {
public void checkFields() throws RemotingCommandException {
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicConfigRequestHeader.java
+3 -2
View File
@@ -17,11 +17,11 @@
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class GetTopicConfigRequestHeader implements CommandCustomHeader {
public class GetTopicConfigRequestHeader extends TopicRequestHeader {
@Override
public void checkFields() throws RemotingCommandException {
}
@@ -29,6 +29,7 @@ public class GetTopicConfigRequestHeader implements CommandCustomHeader {
@CFNotNull
private String topic;
/**
* @return the topic
*/
common/src/main/java/org/apache/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java
+2 -2
View File
@@ -17,11 +17,11 @@
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class GetTopicStatsInfoRequestHeader implements CommandCustomHeader {
public class GetTopicStatsInfoRequestHeader extends TopicRequestHeader {
@CFNotNull
private String topic;
common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java
+6 -2
View File
@@ -20,12 +20,12 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class PullMessageRequestHeader implements CommandCustomHeader {
public class PullMessageRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String consumerGroup;
@CFNotNull
@@ -60,18 +60,22 @@ public class PullMessageRequestHeader implements CommandCustomHeader {
this.consumerGroup = consumerGroup;
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageResponseHeader.java
+11
View File
@@ -22,6 +22,7 @@ package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class PullMessageResponseHeader implements CommandCustomHeader {
@@ -33,6 +34,8 @@ public class PullMessageResponseHeader implements CommandCustomHeader {
private Long minOffset;
@CFNotNull
private Long maxOffset;
@CFNullable
private Long offsetDelta;
@Override
public void checkFields() throws RemotingCommandException {
@@ -69,4 +72,12 @@ public class PullMessageResponseHeader implements CommandCustomHeader {
public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) {
this.suggestWhichBrokerId = suggestWhichBrokerId;
}
public Long getOffsetDelta() {
return offsetDelta;
}
public void setOffsetDelta(Long offsetDelta) {
this.offsetDelta = offsetDelta;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java
+16 -2
View File
@@ -20,11 +20,11 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader {
public class QueryConsumerOffsetRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String consumerGroup;
@CFNotNull
@@ -32,6 +32,8 @@ public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader {
@CFNotNull
private Integer queueId;
private Boolean setZeroIfNotFound;
@Override
public void checkFields() throws RemotingCommandException {
}
@@ -44,19 +46,31 @@ public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader {
this.consumerGroup = consumerGroup;
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
public Boolean getSetZeroIfNotFound() {
return setZeroIfNotFound;
}
public void setSetZeroIfNotFound(Boolean setZeroIfNotFound) {
this.setZeroIfNotFound = setZeroIfNotFound;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java
+6 -2
View File
@@ -20,11 +20,11 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class SearchOffsetRequestHeader implements CommandCustomHeader {
public class SearchOffsetRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String topic;
@CFNotNull
@@ -37,18 +37,22 @@ public class SearchOffsetRequestHeader implements CommandCustomHeader {
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java
+6 -2
View File
@@ -20,12 +20,12 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.annotation.CFNullable;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class SendMessageRequestHeader implements CommandCustomHeader {
public class SendMessageRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String producerGroup;
@CFNotNull
@@ -64,10 +64,12 @@ public class SendMessageRequestHeader implements CommandCustomHeader {
this.producerGroup = producerGroup;
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@@ -88,10 +90,12 @@ public class SendMessageRequestHeader implements CommandCustomHeader {
this.defaultTopicQueueNums = defaultTopicQueueNums;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java
+6 -2
View File
@@ -20,11 +20,11 @@
*/
package org.apache.rocketmq.common.protocol.header;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader {
public class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader {
@CFNotNull
private String consumerGroup;
@CFNotNull
@@ -46,18 +46,22 @@ public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader {
this.consumerGroup = consumerGroup;
}
@Override
public String getTopic() {
return topic;
}
@Override
public void setTopic(String topic) {
this.topic = topic;
}
@Override
public Integer getQueueId() {
return queueId;
}
@Override
public void setQueueId(Integer queueId) {
this.queueId = queueId;
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java
-20
View File
@@ -20,7 +20,6 @@
*/
package org.apache.rocketmq.common.protocol.header.namesrv;
import java.util.Set;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -29,9 +28,6 @@ public class GetRouteInfoRequestHeader implements CommandCustomHeader {
@CFNotNull
private String topic;
private int sysFlag;
private Set<Integer> logicalQueueIdsFilter;
@Override
public void checkFields() throws RemotingCommandException {
}
@@ -43,20 +39,4 @@ public class GetRouteInfoRequestHeader implements CommandCustomHeader {
public void setTopic(String topic) {
this.topic = topic;
}
public int getSysFlag() {
return sysFlag;
}
public void setSysFlag(int sysFlag) {
this.sysFlag = sysFlag;
}
public void setLogicalQueueIdsFilter(Set<Integer> filter) {
this.logicalQueueIdsFilter = filter;
}
public Set<Integer> getLogicalQueueIdsFilter() {
return logicalQueueIdsFilter;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueueRouteData.java
-309
View File
@@ -1,309 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.route;
import com.alibaba.fastjson.annotation.JSONField;
import com.google.common.base.Objects;
import com.google.common.base.Predicate;
import com.google.common.collect.Lists;
import java.util.List;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
/**
* logical queue offset -> message queue offset mapping
*/
public class LogicalQueueRouteData implements Comparable<LogicalQueueRouteData> {
private volatile int logicalQueueIndex = -1; /* -1 means not set */
private volatile long logicalQueueDelta = -1; /* inclusive, -1 means not set, occurred in writeOnly state */
private MessageQueue messageQueue;
private volatile MessageQueueRouteState state = MessageQueueRouteState.Normal;
private volatile long offsetDelta = 0; // valid when Normal/WriteOnly/ReadOnly
private volatile long offsetMax = -1; // exclusive, valid when ReadOnly
private volatile long firstMsgTimeMillis = -1; // valid when ReadOnly
private volatile long lastMsgTimeMillis = -1; // valid when ReadOnly
private String brokerAddr; /* not always set, only used by high availability forward */
public LogicalQueueRouteData() {
}
public LogicalQueueRouteData(int logicalQueueIndex, long logicalQueueDelta,
MessageQueue messageQueue, MessageQueueRouteState state, long offsetDelta, long offsetMax,
long firstMsgTimeMillis,
long lastMsgTimeMillis, String brokerAddr) {
this.logicalQueueIndex = logicalQueueIndex;
this.logicalQueueDelta = logicalQueueDelta;
this.messageQueue = messageQueue;
this.state = state;
this.offsetDelta = offsetDelta;
this.offsetMax = offsetMax;
this.firstMsgTimeMillis = firstMsgTimeMillis;
this.lastMsgTimeMillis = lastMsgTimeMillis;
this.brokerAddr = brokerAddr;
}
public LogicalQueueRouteData(LogicalQueueRouteData queueRouteData) {
copyFrom(queueRouteData);
}
public int getLogicalQueueIndex() {
return logicalQueueIndex;
}
public void setLogicalQueueIndex(int logicalQueueIndex) {
this.logicalQueueIndex = logicalQueueIndex;
}
public long getLogicalQueueDelta() {
return logicalQueueDelta;
}
public void setLogicalQueueDelta(long logicalQueueDelta) {
this.logicalQueueDelta = logicalQueueDelta;
}
public MessageQueue getMessageQueue() {
return messageQueue;
}
public void setMessageQueue(MessageQueue messageQueue) {
this.messageQueue = messageQueue;
}
public MessageQueueRouteState getState() {
return state;
}
@JSONField(serialize = false)
public int getStateOrdinal() {
return state.ordinal();
}
public void setState(MessageQueueRouteState state) {
this.state = state;
}
public String getBrokerAddr() {
return brokerAddr;
}
public void setBrokerAddr(String brokerAddr) {
this.brokerAddr = brokerAddr;
}
public long getOffsetDelta() {
return offsetDelta;
}
public void setOffsetDelta(long offsetDelta) {
this.offsetDelta = offsetDelta;
}
public long getOffsetMax() {
return offsetMax;
}
public void setOffsetMax(long offsetMax) {
this.offsetMax = offsetMax;
}
public long getFirstMsgTimeMillis() {
return firstMsgTimeMillis;
}
public void setFirstMsgTimeMillis(long firstMsgTimeMillis) {
this.firstMsgTimeMillis = firstMsgTimeMillis;
}
public long getLastMsgTimeMillis() {
return lastMsgTimeMillis;
}
public void setLastMsgTimeMillis(long lastMsgTimeMillis) {
this.lastMsgTimeMillis = lastMsgTimeMillis;
}
@Override public String toString() {
return "LogicalQueueRouteData{" +
"logicalQueueIndex=" + logicalQueueIndex +
", logicalQueueDelta=" + logicalQueueDelta +
", messageQueue=" + messageQueue +
", state=" + state +
", offsetDelta=" + offsetDelta +
", offsetMax=" + offsetMax +
", firstMsgTimeMillis=" + firstMsgTimeMillis +
", lastMsgTimeMillis=" + lastMsgTimeMillis +
", brokerAddr='" + brokerAddr + '\'' +
'}';
}
public void copyFrom(LogicalQueueRouteData queueRouteData) {
this.logicalQueueIndex = queueRouteData.logicalQueueIndex;
this.logicalQueueDelta = queueRouteData.logicalQueueDelta;
this.messageQueue = new MessageQueue(queueRouteData.getMessageQueue());
this.state = queueRouteData.state;
this.offsetDelta = queueRouteData.offsetDelta;
this.offsetMax = queueRouteData.offsetMax;
this.firstMsgTimeMillis = queueRouteData.firstMsgTimeMillis;
this.lastMsgTimeMillis = queueRouteData.lastMsgTimeMillis;
this.brokerAddr = queueRouteData.brokerAddr;
}
public long toLogicalQueueOffset(long messageQueueOffset) {
return this.logicalQueueDelta < 0 ? -1 : messageQueueOffset - this.offsetDelta + this.logicalQueueDelta;
}
public long toMessageQueueOffset(long logicalQueueOffset) {
return logicalQueueOffset - this.logicalQueueDelta + this.offsetDelta;
}
@Override public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
LogicalQueueRouteData that = (LogicalQueueRouteData) o;
return logicalQueueIndex == that.logicalQueueIndex && logicalQueueDelta == that.logicalQueueDelta && offsetDelta == that.offsetDelta && offsetMax == that.offsetMax && firstMsgTimeMillis == that.firstMsgTimeMillis && lastMsgTimeMillis == that.lastMsgTimeMillis && Objects.equal(messageQueue, that.messageQueue) && state == that.state && Objects.equal(brokerAddr, that.brokerAddr);
}
@Override public int hashCode() {
return Objects.hashCode(logicalQueueIndex, logicalQueueDelta, messageQueue, state, offsetDelta, offsetMax, firstMsgTimeMillis, lastMsgTimeMillis, brokerAddr);
}
@JSONField(serialize = false)
public long getMessagesCount() {
return this.offsetDelta >= 0 && this.offsetMax >= 0 ? this.offsetMax - this.offsetDelta : 0L;
}
@JSONField(serialize = false)
public boolean isWritable() {
return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.WriteOnly.equals(state);
}
@JSONField(serialize = false)
public boolean isReadable() {
return MessageQueueRouteState.Normal.equals(state) || MessageQueueRouteState.ReadOnly.equals(state);
}
@JSONField(serialize = false)
public boolean isExpired() {
return MessageQueueRouteState.Expired.equals(state);
}
@JSONField(serialize = false)
public boolean isWriteOnly() {
return MessageQueueRouteState.WriteOnly.equals(state);
}
@JSONField(serialize = false)
public int getQueueId() {
return messageQueue.getQueueId();
}
@JSONField(serialize = false)
public String getBrokerName() {
return messageQueue.getBrokerName();
}
@JSONField(serialize = false)
public String getTopic() {
return messageQueue.getTopic();
}
public boolean isSameTo(LogicalQueueRouteData o) {
if (o == null) {
return false;
}
return isSameTo(o.getMessageQueue(), o.offsetDelta);
}
public boolean isSameTo(MessageQueue mq, long offsetDelta) {
return Objects.equal(this.messageQueue, mq) && this.offsetDelta == offsetDelta;
}
/**
* First compare logicalQueueDelta, negative delta must be ordered in the last; then compare state's ordinal; then
* compare messageQueue, nulls first; then compare offsetDelta.
*/
@Override
public int compareTo(LogicalQueueRouteData o) {
long x = this.getLogicalQueueDelta();
long y = o.getLogicalQueueDelta();
int result;
{
if (x >= 0 && y >= 0) {
result = MixAll.compareLong(x, y);
} else if (x < 0 && y < 0) {
result = MixAll.compareLong(-x, -y);
} else if (x < 0) {
// o1 < 0 && o2 >= 0
result = 1;
} else {
// o1 >= 0 && o2 < 0
result = -1;
}
}
if (result == 0) {
result = MixAll.compareInteger(this.state.ordinal(), o.state.ordinal());
}
if (result == 0) {
if (this.messageQueue == null) {
if (o.messageQueue != null) {
result = -1;
}
} else {
if (o.messageQueue != null) {
result = this.messageQueue.compareTo(o.messageQueue);
} else {
result = 1;
}
}
}
if (result == 0) {
result = MixAll.compareLong(this.offsetDelta, o.offsetDelta);
}
return result;
}
public static final Predicate<LogicalQueueRouteData> READABLE_PREDICT = new Predicate<LogicalQueueRouteData>() {
@Override
public boolean apply(LogicalQueueRouteData input) {
return input != null && input.isReadable();
}
};
public List<MessageExt> filterMessages(List<MessageExt> list) {
if (this.offsetMax < 0 || list == null || list.isEmpty()) {
return list;
}
List<MessageExt> result = Lists.newArrayListWithExpectedSize(list.size());
for (MessageExt m : list) {
if (m.getQueueOffset() >= this.offsetMax) {
break;
} else {
result.add(m);
}
}
return result;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfo.java
-87
View File
@@ -1,87 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.route;
import com.alibaba.fastjson.parser.ParserConfig;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer;
public class LogicalQueuesInfo extends HashMap<Integer, List<LogicalQueueRouteData>> {
// TODO whether here needs more fine-grained locks like per logical queue lock?
private final ReadWriteLock lock = new ReentrantReadWriteLock();
public LogicalQueuesInfo() {
super();
}
public LogicalQueuesInfo(Map<Integer, List<LogicalQueueRouteData>> m) {
super(m);
}
public Lock readLock() {
return lock.readLock();
}
public Lock writeLock() {
return lock.writeLock();
}
public void updateLogicalQueueRouteDataList(int logicalQueueIdx,
List<LogicalQueueRouteData> logicalQueueRouteDataList) {
this.writeLock().lock();
try {
logicalQueueRouteDataList = Lists.newLinkedList(logicalQueueRouteDataList);
List<LogicalQueueRouteData> queueRouteDataList = this.get(logicalQueueIdx);
for (LogicalQueueRouteData logicalQueueRouteData : queueRouteDataList) {
for (Iterator<LogicalQueueRouteData> it = logicalQueueRouteDataList.iterator(); it.hasNext(); ) {
LogicalQueueRouteData newQueueRouteData = it.next();
if (Objects.equal(newQueueRouteData.getMessageQueue(), logicalQueueRouteData.getMessageQueue()) && newQueueRouteData.getOffsetDelta() == logicalQueueRouteData.getOffsetDelta()) {
logicalQueueRouteData.copyFrom(newQueueRouteData);
it.remove();
break;
}
}
if (logicalQueueRouteDataList.isEmpty()) {
break;
}
}
for (LogicalQueueRouteData queueRouteData : logicalQueueRouteDataList) {
int idx = Collections.binarySearch(queueRouteDataList, queueRouteData);
if (idx < 0) {
idx = -idx - 1;
}
queueRouteDataList.add(idx, queueRouteData);
}
} finally {
this.writeLock().unlock();
}
}
static {
// workaround https://github.com/alibaba/fastjson/issues/3730
ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfo.class, GenericMapSuperclassDeserializer.INSTANCE);
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/route/LogicalQueuesInfoUnordered.java
-108
View File
@@ -1,108 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.route;
import com.alibaba.fastjson.parser.ParserConfig;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
import org.apache.rocketmq.common.fastjson.GenericMapSuperclassDeserializer;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Only used inside namesrv, between client and namesrv, to reduce cpu usage of namesrv
*/
public class LogicalQueuesInfoUnordered extends ConcurrentHashMap<Integer, Map<LogicalQueuesInfoUnordered.Key, LogicalQueueRouteData>> {
static {
// workaround https://github.com/alibaba/fastjson/issues/3730
ParserConfig.getGlobalInstance().putDeserializer(LogicalQueuesInfoUnordered.class, GenericMapSuperclassDeserializer.INSTANCE);
}
public LogicalQueuesInfoUnordered() {
super();
}
public LogicalQueuesInfoUnordered(int size) {
super(size);
}
public LogicalQueuesInfo toLogicalQueuesInfoOrdered() {
LogicalQueuesInfo logicalQueuesInfoOrdered = new LogicalQueuesInfo();
for (Map.Entry<Integer, Map<Key, LogicalQueueRouteData>> entry : this.entrySet()) {
List<LogicalQueueRouteData> list = Lists.newArrayListWithExpectedSize(entry.getValue().size());
for (LogicalQueueRouteData d : entry.getValue().values()) {
list.add(new LogicalQueueRouteData(d));
}
Collections.sort(list);
logicalQueuesInfoOrdered.put(entry.getKey(), list);
}
return logicalQueuesInfoOrdered;
}
public static class Key {
private final String brokerName;
private final int queueId;
private final long offsetDelta;
private final int hash;
public Key(String brokerName, int queueId, long offsetDelta) {
this.brokerName = brokerName;
this.queueId = queueId;
this.offsetDelta = offsetDelta;
this.hash = Objects.hashCode(brokerName, queueId, this.offsetDelta);
}
public String getBrokerName() {
return brokerName;
}
public int getQueueId() {
return queueId;
}
public long getOffsetDelta() {
return offsetDelta;
}
@Override public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
Key id = (Key) o;
return queueId == id.queueId && offsetDelta == id.offsetDelta && Objects.equal(brokerName, id.brokerName);
}
@Override public int hashCode() {
return hash;
}
@Override public String toString() {
return "Key{" +
"brokerName='" + brokerName + '\'' +
", queueId=" + queueId +
", offsetDelta=" + offsetDelta +
'}';
}
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteData.java
+16 -12
View File
@@ -23,6 +23,9 @@ package org.apache.rocketmq.common.protocol.route;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class TopicRouteData extends RemotingSerializable {
@@ -30,7 +33,8 @@ public class TopicRouteData extends RemotingSerializable {
private List<QueueData> queueDatas;
private List<BrokerData> brokerDatas;
private HashMap<String/* brokerAddr */, List<String>/* Filter Server */> filterServerTable;
private LogicalQueuesInfo logicalQueuesInfo;
//It could be null or empty
private Map<String/*brokerName*/, TopicQueueMappingInfo> topicQueueMappingByBroker;
public TopicRouteData() {
}
@@ -53,8 +57,8 @@ public class TopicRouteData extends RemotingSerializable {
this.filterServerTable.putAll(topicRouteData.filterServerTable);
}
if (topicRouteData.logicalQueuesInfo != null) {
this.logicalQueuesInfo = new LogicalQueuesInfo(topicRouteData.logicalQueuesInfo);
if (topicRouteData.topicQueueMappingByBroker != null) {
this.topicQueueMappingByBroker = new HashMap<String, TopicQueueMappingInfo>(topicRouteData.topicQueueMappingByBroker);
}
}
@@ -90,12 +94,12 @@ public class TopicRouteData extends RemotingSerializable {
this.orderTopicConf = orderTopicConf;
}
public LogicalQueuesInfo getLogicalQueuesInfo() {
return logicalQueuesInfo;
public Map<String, TopicQueueMappingInfo> getTopicQueueMappingByBroker() {
return topicQueueMappingByBroker;
}
public void setLogicalQueuesInfo(LogicalQueuesInfo logicalQueuesInfo) {
this.logicalQueuesInfo = logicalQueuesInfo;
public void setTopicQueueMappingByBroker(Map<String, TopicQueueMappingInfo> topicQueueMappingByBroker) {
this.topicQueueMappingByBroker = topicQueueMappingByBroker;
}
@Override
@@ -106,7 +110,7 @@ public class TopicRouteData extends RemotingSerializable {
result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode());
result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode());
result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode());
result = prime * result + ((logicalQueuesInfo == null) ? 0 : logicalQueuesInfo.hashCode());
result = prime * result + ((topicQueueMappingByBroker == null) ? 0 : topicQueueMappingByBroker.hashCode());
return result;
}
@@ -139,10 +143,10 @@ public class TopicRouteData extends RemotingSerializable {
return false;
} else if (!filterServerTable.equals(other.filterServerTable))
return false;
if (logicalQueuesInfo == null) {
if (other.logicalQueuesInfo != null)
if (topicQueueMappingByBroker == null) {
if (other.topicQueueMappingByBroker != null)
return false;
} else if (!logicalQueuesInfo.equals(other.logicalQueuesInfo))
} else if (!topicQueueMappingByBroker.equals(other.topicQueueMappingByBroker))
return false;
return true;
}
@@ -150,6 +154,6 @@ public class TopicRouteData extends RemotingSerializable {
@Override
public String toString() {
return "TopicRouteData [orderTopicConf=" + orderTopicConf + ", queueDatas=" + queueDatas
+ ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + ", logicalQueuesInfo=" + logicalQueuesInfo + "]";
+ ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + ", topicQueueMappingInfoTable=" + topicQueueMappingByBroker + "]";
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/route/TopicRouteDataNameSrv.java
-64
View File
@@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.route;
import com.google.common.base.Objects;
public class TopicRouteDataNameSrv extends TopicRouteData {
private LogicalQueuesInfoUnordered logicalQueuesInfoUnordered;
public TopicRouteDataNameSrv() {
}
public LogicalQueuesInfoUnordered getLogicalQueuesInfoUnordered() {
return logicalQueuesInfoUnordered;
}
public void setLogicalQueuesInfoUnordered(
LogicalQueuesInfoUnordered logicalQueuesInfoUnordered) {
this.logicalQueuesInfoUnordered = logicalQueuesInfoUnordered;
}
@Override public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
if (!super.equals(o))
return false;
TopicRouteDataNameSrv srv = (TopicRouteDataNameSrv) o;
return Objects.equal(logicalQueuesInfoUnordered, srv.logicalQueuesInfoUnordered);
}
@Override public int hashCode() {
return Objects.hashCode(super.hashCode(), logicalQueuesInfoUnordered);
}
@Override public String toString() {
return "TopicRouteDataNameSrv{" +
"logicalQueuesInfoUnordered=" + logicalQueuesInfoUnordered +
"} " + super.toString();
}
public TopicRouteData toTopicRouteData() {
TopicRouteData topicRouteData = new TopicRouteData(this);
if (this.logicalQueuesInfoUnordered != null) {
topicRouteData.setLogicalQueuesInfo(this.logicalQueuesInfoUnordered.toLogicalQueuesInfoOrdered());
}
return topicRouteData;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/ClientMetadata.java
+176
View File
@@ -0,0 +1,176 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingInfo;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.body.ClusterInfo;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils;
import org.apache.rocketmq.logging.InternalLogger;
import org.apache.rocketmq.logging.InternalLoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class ClientMetadata {
private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
private final ConcurrentMap<String/* Topic */, TopicRouteData> topicRouteTable = new ConcurrentHashMap<String, TopicRouteData>();
private final ConcurrentMap<String/* Topic */, ConcurrentMap<MessageQueue, String/*brokerName*/>> topicEndPointsTable = new ConcurrentHashMap<String, ConcurrentMap<MessageQueue, String>>();
private final ConcurrentMap<String/* Broker Name */, HashMap<Long/* brokerId */, String/* address */>> brokerAddrTable =
new ConcurrentHashMap<String, HashMap<Long, String>>();
private final ConcurrentMap<String/* Broker Name */, HashMap<String/* address */, Integer>> brokerVersionTable =
new ConcurrentHashMap<String, HashMap<String, Integer>>();
public void freshTopicRoute(String topic, TopicRouteData topicRouteData) {
if (topic == null
|| topicRouteData == null) {
return;
}
TopicRouteData old = this.topicRouteTable.get(topic);
if (!topicRouteDataIsChange(old, topicRouteData)) {
return ;
}
{
for (BrokerData bd : topicRouteData.getBrokerDatas()) {
this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs());
}
}
{
ConcurrentMap<MessageQueue, String> mqEndPoints = topicRouteData2EndpointsForStaticTopic(topic, topicRouteData);
if (mqEndPoints != null
&& !mqEndPoints.isEmpty()) {
topicEndPointsTable.put(topic, mqEndPoints);
}
}
}
public static boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) {
if (olddata == null || nowdata == null)
return true;
TopicRouteData old = new TopicRouteData(olddata);
TopicRouteData now = new TopicRouteData(nowdata);
Collections.sort(old.getQueueDatas());
Collections.sort(old.getBrokerDatas());
Collections.sort(now.getQueueDatas());
Collections.sort(now.getBrokerDatas());
return !old.equals(now);
}
public String getBrokerNameFromMessageQueue(final MessageQueue mq) {
if (topicEndPointsTable.get(mq.getTopic()) != null
&& !topicEndPointsTable.get(mq.getTopic()).isEmpty()) {
return topicEndPointsTable.get(mq.getTopic()).get(mq);
}
return mq.getBrokerName();
}
public void refreshClusterInfo(ClusterInfo clusterInfo) {
if (clusterInfo == null
|| clusterInfo.getBrokerAddrTable() == null) {
return;
}
for (Map.Entry<String, BrokerData> entry : clusterInfo.getBrokerAddrTable().entrySet()) {
brokerAddrTable.put(entry.getKey(), entry.getValue().getBrokerAddrs());
}
}
public String findMasterBrokerAddr(String brokerName) {
if (!brokerAddrTable.containsKey(brokerName)) {
return null;
}
return brokerAddrTable.get(brokerName).get(MixAll.MASTER_ID);
}
public ConcurrentMap<String, HashMap<Long, String>> getBrokerAddrTable() {
return brokerAddrTable;
}
public static ConcurrentMap<MessageQueue, String> topicRouteData2EndpointsForStaticTopic(final String topic, final TopicRouteData route) {
if (route.getTopicQueueMappingByBroker() == null
|| route.getTopicQueueMappingByBroker().isEmpty()) {
return new ConcurrentHashMap<MessageQueue, String>();
}
ConcurrentMap<MessageQueue, String> mqEndPointsOfBroker = new ConcurrentHashMap<MessageQueue, String>();
Map<String, Map<String, TopicQueueMappingInfo>> mappingInfosByScope = new HashMap<String, Map<String, TopicQueueMappingInfo>>();
for (Map.Entry<String, TopicQueueMappingInfo> entry : route.getTopicQueueMappingByBroker().entrySet()) {
TopicQueueMappingInfo info = entry.getValue();
String scope = info.getScope();
if (scope != null) {
if (!mappingInfosByScope.containsKey(scope)) {
mappingInfosByScope.put(scope, new HashMap<String, TopicQueueMappingInfo>());
}
mappingInfosByScope.get(scope).put(entry.getKey(), entry.getValue());
}
}
for (Map.Entry<String, Map<String, TopicQueueMappingInfo>> mapEntry : mappingInfosByScope.entrySet()) {
String scope = mapEntry.getKey();
Map<String, TopicQueueMappingInfo> topicQueueMappingInfoMap = mapEntry.getValue();
ConcurrentMap<MessageQueue, TopicQueueMappingInfo> mqEndPoints = new ConcurrentHashMap<MessageQueue, TopicQueueMappingInfo>();
List<Map.Entry<String, TopicQueueMappingInfo>> mappingInfos = new ArrayList<Map.Entry<String, TopicQueueMappingInfo>>(topicQueueMappingInfoMap.entrySet());
Collections.sort(mappingInfos, new Comparator<Map.Entry<String, TopicQueueMappingInfo>>() {
@Override
public int compare(Map.Entry<String, TopicQueueMappingInfo> o1, Map.Entry<String, TopicQueueMappingInfo> o2) {
return (int) (o2.getValue().getEpoch() - o1.getValue().getEpoch());
}
});
int maxTotalNums = 0;
long maxTotalNumOfEpoch = -1;
for (Map.Entry<String, TopicQueueMappingInfo> entry : mappingInfos) {
TopicQueueMappingInfo info = entry.getValue();
if (info.getEpoch() >= maxTotalNumOfEpoch && info.getTotalQueues() > maxTotalNums) {
maxTotalNums = info.getTotalQueues();
}
for (Map.Entry<Integer, Integer> idEntry : entry.getValue().getCurrIdMap().entrySet()) {
int globalId = idEntry.getKey();
MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(info.getScope()), globalId);
TopicQueueMappingInfo oldInfo = mqEndPoints.get(mq);
if (oldInfo == null || oldInfo.getEpoch() <= info.getEpoch()) {
mqEndPoints.put(mq, info);
}
}
}
//accomplish the static logic queues
for (int i = 0; i < maxTotalNums; i++) {
MessageQueue mq = new MessageQueue(topic, TopicQueueMappingUtils.getMockBrokerName(scope), i);
if (!mqEndPoints.containsKey(mq)) {
mqEndPointsOfBroker.put(mq, MixAll.LOGICAL_QUEUE_MOCK_BROKER_NAME_NOT_EXIST);
} else {
mqEndPointsOfBroker.put(mq, mqEndPoints.get(mq).getBname());
}
}
}
return mqEndPointsOfBroker;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/RequestBuilder.java
+81
View File
@@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader;
import java.util.HashMap;
import java.util.Map;
public class RequestBuilder {
private static Map<Integer, Class> requestCodeMap = new HashMap<Integer, Class>();
static {
requestCodeMap.put(RequestCode.PULL_MESSAGE, PullMessageRequestHeader.class);
}
public static RpcRequestHeader buildCommonRpcHeader(int requestCode, String destBrokerName) {
return buildCommonRpcHeader(requestCode, null, destBrokerName);
}
public static RpcRequestHeader buildCommonRpcHeader(int requestCode, Boolean oneway, String destBrokerName) {
Class requestHeaderClass = requestCodeMap.get(requestCode);
if (requestHeaderClass == null) {
throw new UnsupportedOperationException("unknown " + requestCode);
}
try {
RpcRequestHeader requestHeader = (RpcRequestHeader) requestHeaderClass.newInstance();
requestHeader.setOway(oneway);
requestHeader.setBname(destBrokerName);
return requestHeader;
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq) {
return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), null);
}
public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, MessageQueue mq, Boolean logic) {
return buildTopicQueueRequestHeader(requestCode, null, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic);
}
public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, Boolean oneway, MessageQueue mq, Boolean logic) {
return buildTopicQueueRequestHeader(requestCode, oneway, mq.getBrokerName(), mq.getTopic(), mq.getQueueId(), logic);
}
public static TopicQueueRequestHeader buildTopicQueueRequestHeader(int requestCode, Boolean oneway, String destBrokerName, String topic, int queueId, Boolean logic) {
Class requestHeaderClass = requestCodeMap.get(requestCode);
if (requestHeaderClass == null) {
throw new UnsupportedOperationException("unknown " + requestCode);
}
try {
TopicQueueRequestHeader requestHeader = (TopicQueueRequestHeader) requestHeaderClass.newInstance();
requestHeader.setOway(oneway);
requestHeader.setBname(destBrokerName);
requestHeader.setTopic(topic);
requestHeader.setQueueId(queueId);
requestHeader.setLo(logic);
return requestHeader;
} catch (Throwable t) {
throw new RuntimeException(t);
}
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcClient.java
+35
View File
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.common.message.MessageQueue;
import java.util.concurrent.Future;
public interface RpcClient {
//common invoke paradigm, the logic remote addr is defined in "bname" field of request
//For oneway request, the sign is labeled in request, and do not need an another method named "invokeOneway"
//For one
Future<RpcResponse> invoke(RpcRequest request, long timeoutMs) throws RpcException;
//For rocketmq, most requests are corresponded to MessageQueue
//And for LogicQueue, the broker name is mocked, the physical addr could only be defined by MessageQueue
Future<RpcResponse> invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException;
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientHook.java
+27
View File
@@ -0,0 +1,27 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
public abstract class RpcClientHook {
//if the return is not null, return it
public abstract RpcResponse beforeRequest(RpcRequest rpcRequest) throws RpcException;
//if the return is not null, return it
public abstract RpcResponse afterResponse(RpcResponse rpcResponse) throws RpcException;
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientImpl.java
+317
View File
@@ -0,0 +1,317 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import io.netty.util.concurrent.ImmediateEventExecutor;
import io.netty.util.concurrent.Promise;
import org.apache.rocketmq.common.admin.TopicStatsTable;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.RequestCode;
import org.apache.rocketmq.common.protocol.ResponseCode;
import org.apache.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader;
import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.PullMessageResponseHeader;
import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader;
import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader;
import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping;
import org.apache.rocketmq.remoting.InvokeCallback;
import org.apache.rocketmq.remoting.RemotingClient;
import org.apache.rocketmq.remoting.netty.ResponseFuture;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Future;
public class RpcClientImpl implements RpcClient {
private ClientMetadata clientMetadata;
private RemotingClient remotingClient;
private List<RpcClientHook> clientHookList = new ArrayList<RpcClientHook>();
public RpcClientImpl(ClientMetadata clientMetadata, RemotingClient remotingClient) {
this.clientMetadata = clientMetadata;
this.remotingClient = remotingClient;
}
public void registerHook(RpcClientHook hook) {
clientHookList.add(hook);
}
@Override
public Future<RpcResponse> invoke(MessageQueue mq, RpcRequest request, long timeoutMs) throws RpcException {
String bname = clientMetadata.getBrokerNameFromMessageQueue(mq);
request.getHeader().setBname(bname);
return invoke(request, timeoutMs);
}
public Promise<RpcResponse> createResponseFuture() {
return ImmediateEventExecutor.INSTANCE.newPromise();
}
@Override
public Future<RpcResponse> invoke(RpcRequest request, long timeoutMs) throws RpcException {
if (clientHookList.size() > 0) {
for (RpcClientHook rpcClientHook: clientHookList) {
RpcResponse response = rpcClientHook.beforeRequest(request);
if (response != null) {
//For 1.6, there is not easy-to-use future impl
return createResponseFuture().setSuccess(response);
}
}
}
String addr = getBrokerAddrByNameOrException(request.getHeader().bname);
Promise<RpcResponse> rpcResponsePromise = null;
try {
switch (request.getCode()) {
case RequestCode.PULL_MESSAGE:
rpcResponsePromise = handlePullMessage(addr, request, timeoutMs);
break;
case RequestCode.GET_MIN_OFFSET:
rpcResponsePromise = handleGetMinOffset(addr, request, timeoutMs);
break;
case RequestCode.GET_MAX_OFFSET:
rpcResponsePromise = handleGetMaxOffset(addr, request, timeoutMs);
break;
case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP:
rpcResponsePromise = handleSearchOffset(addr, request, timeoutMs);
break;
case RequestCode.GET_EARLIEST_MSG_STORETIME:
rpcResponsePromise = handleGetEarliestMsgStoretime(addr, request, timeoutMs);
break;
case RequestCode.QUERY_CONSUMER_OFFSET:
rpcResponsePromise = handleQueryConsumerOffset(addr, request, timeoutMs);
break;
case RequestCode.GET_TOPIC_STATS_INFO:
rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicStatsTable.class);
break;
case RequestCode.GET_TOPIC_CONFIG:
rpcResponsePromise = handleCommonBodyRequest(addr, request, timeoutMs, TopicConfigAndQueueMapping.class);
break;
default:
throw new RpcException(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, "Unknown request code " + request.getCode());
}
} catch (RpcException rpcException) {
throw rpcException;
} catch (Exception e) {
throw new RpcException(ResponseCode.RPC_UNKNOWN, "error from remoting layer", e);
}
return rpcResponsePromise;
}
private String getBrokerAddrByNameOrException(String bname) throws RpcException {
String addr = this.clientMetadata.findMasterBrokerAddr(bname);
if (addr == null) {
throw new RpcException(ResponseCode.SYSTEM_ERROR, "cannot find addr for broker " + bname);
}
return addr;
}
private void processFailedResponse(String addr, RemotingCommand requestCommand, ResponseFuture responseFuture, Promise<RpcResponse> rpcResponsePromise) {
RemotingCommand responseCommand = responseFuture.getResponseCommand();
if (responseCommand != null) {
//this should not happen
return;
}
int errorCode = ResponseCode.RPC_UNKNOWN;
String errorMessage = null;
if (!responseFuture.isSendRequestOK()) {
errorCode = ResponseCode.RPC_SEND_TO_CHANNEL_FAILED;
errorMessage = "send request failed to " + addr + ". Request: " + requestCommand;
} else if (responseFuture.isTimeout()) {
errorCode = ResponseCode.RPC_TIME_OUT;
errorMessage = "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + requestCommand;
} else {
errorMessage = "unknown reason. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand;
}
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(errorCode, errorMessage)));
}
public Promise<RpcResponse> handlePullMessage(final String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
InvokeCallback callback = new InvokeCallback() {
@Override
public void operationComplete(ResponseFuture responseFuture) {
RemotingCommand responseCommand = responseFuture.getResponseCommand();
if (responseCommand == null) {
processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise);
return;
}
try {
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS:
case ResponseCode.PULL_NOT_FOUND:
case ResponseCode.PULL_RETRY_IMMEDIATELY:
case ResponseCode.PULL_OFFSET_MOVED:
PullMessageResponseHeader responseHeader =
(PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
default:
RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code"));
rpcResponsePromise.setSuccess(rpcResponse);
}
} catch (Exception e) {
String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand;
RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e));
rpcResponsePromise.setSuccess(rpcResponse);
}
}
};
this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback);
return rpcResponsePromise;
}
public Promise<RpcResponse> handleSearchOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
SearchOffsetResponseHeader responseHeader =
(SearchOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(SearchOffsetResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
public Promise<RpcResponse> handleQueryConsumerOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
QueryConsumerOffsetResponseHeader responseHeader =
(QueryConsumerOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
break;
}
case ResponseCode.QUERY_NOT_FOUND: {
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), null, null));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
public Promise<RpcResponse> handleCommonBodyRequest(final String addr, RpcRequest rpcRequest, long timeoutMillis, Class bodyClass) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
rpcResponsePromise.setSuccess(new RpcResponse(ResponseCode.SUCCESS, null, RemotingSerializable.decode(responseCommand.getBody(), bodyClass)));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
public Promise<RpcResponse> handleGetMinOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
GetMinOffsetResponseHeader responseHeader =
(GetMinOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
public Promise<RpcResponse> handleGetMaxOffset(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
GetMaxOffsetResponseHeader responseHeader =
(GetMaxOffsetResponseHeader) responseCommand.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
public Promise<RpcResponse> handleGetEarliestMsgStoretime(String addr, RpcRequest rpcRequest, long timeoutMillis) throws Exception {
final Promise<RpcResponse> rpcResponsePromise = createResponseFuture();
RemotingCommand requestCommand = RpcClientUtils.createCommandForRpcRequest(rpcRequest);
RemotingCommand responseCommand = this.remotingClient.invokeSync(addr, requestCommand, timeoutMillis);
assert responseCommand != null;
switch (responseCommand.getCode()) {
case ResponseCode.SUCCESS: {
GetEarliestMsgStoretimeResponseHeader responseHeader =
(GetEarliestMsgStoretimeResponseHeader) responseCommand.decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class);
rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody()));
break;
}
default: {
rpcResponsePromise.setSuccess(new RpcResponse(new RpcException(responseCommand.getCode(), "unknown remote error")));
}
}
return rpcResponsePromise;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcClientUtils.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import java.nio.ByteBuffer;
public class RpcClientUtils {
public static RemotingCommand createCommandForRpcRequest(RpcRequest rpcRequest) {
RemotingCommand cmd = RemotingCommand.createRequestCommand(rpcRequest.getCode(), rpcRequest.getHeader());
cmd.setBody(encodeBody(rpcRequest.getBody()));
return cmd;
}
public static RemotingCommand createCommandForRpcResponse(RpcResponse rpcResponse) {
RemotingCommand cmd = RemotingCommand.createResponseCommandWithHeader(rpcResponse.getCode(), rpcResponse.getHeader());
cmd.setRemark(rpcResponse.getException() == null ? "" : rpcResponse.getException().getMessage());
cmd.setBody(encodeBody(rpcResponse.getBody()));
return cmd;
}
public static byte[] encodeBody(Object body) {
if (body == null) {
return null;
}
if (body instanceof byte[]) {
return (byte[])body;
} else if (body instanceof RemotingSerializable) {
return ((RemotingSerializable) body).encode();
} else if (body instanceof ByteBuffer) {
ByteBuffer buffer = (ByteBuffer)body;
buffer.mark();
byte[] data = new byte[buffer.remaining()];
buffer.get(data);
buffer.reset();
return data;
} else {
throw new RuntimeException("Unsupported body type " + body.getClass());
}
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/body/SealTopicLogicalQueueRequestBody.java → common/src/main/java/org/apache/rocketmq/common/rpc/RpcException.java
+14 -23
View File
@@ -14,36 +14,27 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.body;
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.remoting.exception.RemotingException;
public class SealTopicLogicalQueueRequestBody extends RemotingSerializable {
private String topic;
private int queueId;
private int logicalQueueIndex;
public String getTopic() {
return topic;
public class RpcException extends RemotingException {
private int errorCode;
public RpcException(int errorCode, String message) {
super(message);
this.errorCode = errorCode;
}
public void setTopic(String topic) {
this.topic = topic;
public RpcException(int errorCode, String message, Throwable cause) {
super(message, cause);
this.errorCode = errorCode;
}
public int getQueueId() {
return queueId;
public int getErrorCode() {
return errorCode;
}
public void setQueueId(int queueId) {
this.queueId = queueId;
}
public int getLogicalQueueIndex() {
return logicalQueueIndex;
}
public void setLogicalQueueIndex(int logicalQueueIndex) {
this.logicalQueueIndex = logicalQueueIndex;
public void setErrorCode(int errorCode) {
this.errorCode = errorCode;
}
}
common/src/main/java/org/apache/rocketmq/common/protocol/header/QueryTopicLogicalQueueMappingRequestHeader.java → common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequest.java
+17 -13
View File
@@ -14,24 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.protocol.header;
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.remoting.CommandCustomHeader;
import org.apache.rocketmq.remoting.annotation.CFNotNull;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
public class RpcRequest {
int code;
private RpcRequestHeader header;
private Object body;
public class QueryTopicLogicalQueueMappingRequestHeader implements CommandCustomHeader {
@CFNotNull
private String topic;
@Override public void checkFields() throws RemotingCommandException {
public RpcRequest(int code, RpcRequestHeader header, Object body) {
this.code = code;
this.header = header;
this.body = body;
}
public String getTopic() {
return topic;
public RpcRequestHeader getHeader() {
return header;
}
public void setTopic(String topic) {
this.topic = topic;
public Object getBody() {
return body;
}
public int getCode() {
return code;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcRequestHeader.java
+62
View File
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.remoting.CommandCustomHeader;
public abstract class RpcRequestHeader implements CommandCustomHeader {
//the namespace name
protected String ns;
//if the data has been namespaced
protected Boolean nsd;
//the abstract remote addr name, usually the physical broker name
protected String bname;
//oneway
protected Boolean oway;
public String getBname() {
return bname;
}
public void setBname(String bname) {
this.bname = bname;
}
public Boolean getOway() {
return oway;
}
public void setOway(Boolean oway) {
this.oway = oway;
}
public String getNs() {
return ns;
}
public void setNs(String ns) {
this.ns = ns;
}
public Boolean getNsd() {
return nsd;
}
public void setNsd(Boolean nsd) {
this.nsd = nsd;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/RpcResponse.java
+70
View File
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
import org.apache.rocketmq.remoting.CommandCustomHeader;
public class RpcResponse {
private int code;
private CommandCustomHeader header;
private Object body;
public RpcException exception;
public RpcResponse() {
}
public RpcResponse(int code, CommandCustomHeader header, Object body) {
this.code = code;
this.header = header;
this.body = body;
}
public RpcResponse(RpcException rpcException) {
this.code = rpcException.getErrorCode();
this.exception = rpcException;
}
public int getCode() {
return code;
}
public CommandCustomHeader getHeader() {
return header;
}
public void setHeader(CommandCustomHeader header) {
this.header = header;
}
public Object getBody() {
return body;
}
public void setBody(Object body) {
this.body = body;
}
public RpcException getException() {
return exception;
}
public void setException(RpcException exception) {
this.exception = exception;
}
}
common/src/main/java/org/apache/rocketmq/common/rpc/TopicQueueRequestHeader.java
+24
View File
@@ -0,0 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
public abstract class TopicQueueRequestHeader extends TopicRequestHeader {
public abstract Integer getQueueId();
public abstract void setQueueId(Integer queueId);
}
common/src/main/java/org/apache/rocketmq/common/rpc/TopicRequestHeader.java
+32
View File
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.rpc;
public abstract class TopicRequestHeader extends RpcRequestHeader {
//logical
protected Boolean lo;
public abstract String getTopic();
public abstract void setTopic(String topic);
public Boolean getLo() {
return lo;
}
public void setLo(Boolean lo) {
this.lo = lo;
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/LogicQueueMappingItem.java
+212
View File
@@ -0,0 +1,212 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
public class LogicQueueMappingItem extends RemotingSerializable {
private int gen; // immutable
private int queueId; //, immutable
private String bname; //important, immutable
private long logicOffset; // the start of the logic offset, important, can be changed by command only once
private long startOffset; // the start of the physical offset, should always be 0, immutable
private long endOffset = -1; // the end of the physical offset, excluded, revered -1, mutable
private long timeOfStart = -1; // mutable, reserved
private long timeOfEnd = -1; // mutable, reserved
//make sure it has a default constructor
public LogicQueueMappingItem() {
}
public LogicQueueMappingItem(int gen, int queueId, String bname, long logicOffset, long startOffset, long endOffset, long timeOfStart, long timeOfEnd) {
this.gen = gen;
this.queueId = queueId;
this.bname = bname;
this.logicOffset = logicOffset;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.timeOfStart = timeOfStart;
this.timeOfEnd = timeOfEnd;
}
//should only be user in sendMessage and getMinOffset
public long computeStaticQueueOffsetLoosely(long physicalQueueOffset) {
//consider the newly mapped item
if (logicOffset < 0) {
return -1;
}
if (physicalQueueOffset < startOffset) {
return logicOffset;
}
if (endOffset >= startOffset
&& endOffset < physicalQueueOffset) {
return logicOffset + (endOffset - startOffset);
}
return logicOffset + (physicalQueueOffset - startOffset);
}
public long computeStaticQueueOffsetStrictly(long physicalQueueOffset) {
assert logicOffset >= 0;
if (physicalQueueOffset < startOffset) {
return logicOffset;
}
return logicOffset + (physicalQueueOffset - startOffset);
}
public long computePhysicalQueueOffset(long staticQueueOffset) {
return (staticQueueOffset - logicOffset) + startOffset;
}
public long computeMaxStaticQueueOffset() {
if (endOffset >= startOffset) {
return logicOffset + endOffset - startOffset;
} else {
return logicOffset;
}
}
public boolean checkIfEndOffsetDecided() {
//if the endOffset == startOffset, then the item should be deleted
return endOffset > startOffset;
}
public boolean checkIfLogicoffsetDecided() {
return logicOffset >= 0;
}
public long computeOffsetDelta() {
return logicOffset - startOffset;
}
public int getGen() {
return gen;
}
public int getQueueId() {
return queueId;
}
public String getBname() {
return bname;
}
public long getLogicOffset() {
return logicOffset;
}
public long getStartOffset() {
return startOffset;
}
public long getEndOffset() {
return endOffset;
}
public long getTimeOfStart() {
return timeOfStart;
}
public long getTimeOfEnd() {
return timeOfEnd;
}
public void setLogicOffset(long logicOffset) {
this.logicOffset = logicOffset;
}
public void setEndOffset(long endOffset) {
this.endOffset = endOffset;
}
public void setTimeOfStart(long timeOfStart) {
this.timeOfStart = timeOfStart;
}
public void setTimeOfEnd(long timeOfEnd) {
this.timeOfEnd = timeOfEnd;
}
public void setGen(int gen) {
this.gen = gen;
}
public void setQueueId(int queueId) {
this.queueId = queueId;
}
public void setBname(String bname) {
this.bname = bname;
}
public void setStartOffset(long startOffset) {
this.startOffset = startOffset;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof LogicQueueMappingItem)) return false;
LogicQueueMappingItem item = (LogicQueueMappingItem) o;
return new EqualsBuilder()
.append(gen, item.gen)
.append(queueId, item.queueId)
.append(logicOffset, item.logicOffset)
.append(startOffset, item.startOffset)
.append(endOffset, item.endOffset)
.append(timeOfStart, item.timeOfStart)
.append(timeOfEnd, item.timeOfEnd)
.append(bname, item.bname)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.append(gen)
.append(queueId)
.append(bname)
.append(logicOffset)
.append(startOffset)
.append(endOffset)
.append(timeOfStart)
.append(timeOfEnd)
.toHashCode();
}
@Override
public String toString() {
return "LogicQueueMappingItem{" +
"gen=" + gen +
", queueId=" + queueId +
", bname='" + bname + '\'' +
", logicOffset=" + logicOffset +
", startOffset=" + startOffset +
", endOffset=" + endOffset +
", timeOfStart=" + timeOfStart +
", timeOfEnd=" + timeOfEnd +
'}';
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicConfigAndQueueMapping.java
+63
View File
@@ -0,0 +1,63 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.rocketmq.common.TopicConfig;
public class TopicConfigAndQueueMapping extends TopicConfig {
private TopicQueueMappingDetail mappingDetail;
public TopicConfigAndQueueMapping() {
}
public TopicConfigAndQueueMapping(TopicConfig topicConfig, TopicQueueMappingDetail mappingDetail) {
super(topicConfig);
this.mappingDetail = mappingDetail;
}
public TopicQueueMappingDetail getMappingDetail() {
return mappingDetail;
}
public void setMappingDetail(TopicQueueMappingDetail mappingDetail) {
this.mappingDetail = mappingDetail;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof TopicConfigAndQueueMapping)) return false;
TopicConfigAndQueueMapping that = (TopicConfigAndQueueMapping) o;
return new EqualsBuilder()
.appendSuper(super.equals(o))
.append(mappingDetail, that.mappingDetail)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.appendSuper(super.hashCode())
.append(mappingDetail)
.toHashCode();
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingContext.java
+99
View File
@@ -0,0 +1,99 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import com.google.common.collect.ImmutableList;
import java.util.List;
public class TopicQueueMappingContext {
private String topic;
private Integer globalId;
private TopicQueueMappingDetail mappingDetail;
private List<LogicQueueMappingItem> mappingItemList;
private LogicQueueMappingItem leaderItem;
private LogicQueueMappingItem currentItem;
public TopicQueueMappingContext(String topic, Integer globalId, TopicQueueMappingDetail mappingDetail, List<LogicQueueMappingItem> mappingItemList, LogicQueueMappingItem leaderItem) {
this.topic = topic;
this.globalId = globalId;
this.mappingDetail = mappingDetail;
this.mappingItemList = mappingItemList;
this.leaderItem = leaderItem;
}
public boolean isLeader() {
return leaderItem != null && leaderItem.getBname().equals(mappingDetail.getBname());
}
public String getTopic() {
return topic;
}
public void setTopic(String topic) {
this.topic = topic;
}
public Integer getGlobalId() {
return globalId;
}
public void setGlobalId(Integer globalId) {
this.globalId = globalId;
}
public TopicQueueMappingDetail getMappingDetail() {
return mappingDetail;
}
public void setMappingDetail(TopicQueueMappingDetail mappingDetail) {
this.mappingDetail = mappingDetail;
}
public List<LogicQueueMappingItem> getMappingItemList() {
return mappingItemList;
}
public void setMappingItemList(ImmutableList<LogicQueueMappingItem> mappingItemList) {
this.mappingItemList = mappingItemList;
}
public LogicQueueMappingItem getLeaderItem() {
return leaderItem;
}
public void setLeaderItem(LogicQueueMappingItem leaderItem) {
this.leaderItem = leaderItem;
}
public LogicQueueMappingItem getCurrentItem() {
return currentItem;
}
public void setCurrentItem(LogicQueueMappingItem currentItem) {
this.currentItem = currentItem;
}
public void setMappingItemList(List<LogicQueueMappingItem> mappingItemList) {
this.mappingItemList = mappingItemList;
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingDetail.java
+144
View File
@@ -0,0 +1,144 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class TopicQueueMappingDetail extends TopicQueueMappingInfo {
// the mapping info in current broker, do not register to nameserver
// make sure this value is not null
private ConcurrentMap<Integer/*global id*/, List<LogicQueueMappingItem>> hostedQueues = new ConcurrentHashMap<Integer, List<LogicQueueMappingItem>>();
//make sure there is a default constructor
public TopicQueueMappingDetail() {
}
public TopicQueueMappingDetail(String topic, int totalQueues, String bname, long epoch) {
super(topic, totalQueues, bname, epoch);
}
public static boolean putMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId, List<LogicQueueMappingItem> mappingInfo) {
if (mappingInfo.isEmpty()) {
return true;
}
mappingDetail.hostedQueues.put(globalId, mappingInfo);
return true;
}
public static List<LogicQueueMappingItem> getMappingInfo(TopicQueueMappingDetail mappingDetail, Integer globalId) {
return mappingDetail.hostedQueues.get(globalId);
}
public static ConcurrentMap<Integer, Integer> buildIdMap(TopicQueueMappingDetail mappingDetail, int level) {
//level 0 means current leader in this broker
//level 1 means previous leader in this broker, reserved for
assert level == LEVEL_0 ;
if (mappingDetail.hostedQueues == null || mappingDetail.hostedQueues.isEmpty()) {
return new ConcurrentHashMap<Integer, Integer>();
}
ConcurrentMap<Integer, Integer> tmpIdMap = new ConcurrentHashMap<Integer, Integer>();
for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry: mappingDetail.hostedQueues.entrySet()) {
Integer globalId = entry.getKey();
List<LogicQueueMappingItem> items = entry.getValue();
if (level == LEVEL_0
&& items.size() >= 1) {
LogicQueueMappingItem curr = items.get(items.size() - 1);
if (mappingDetail.bname.equals(curr.getBname())) {
tmpIdMap.put(globalId, curr.getQueueId());
}
}
}
return tmpIdMap;
}
public static long computeMaxOffsetFromMapping(TopicQueueMappingDetail mappingDetail, Integer globalId) {
List<LogicQueueMappingItem> mappingItems = getMappingInfo(mappingDetail, globalId);
if (mappingItems == null
|| mappingItems.isEmpty()) {
return -1;
}
LogicQueueMappingItem item = mappingItems.get(mappingItems.size() - 1);
return item.computeMaxStaticQueueOffset();
}
public static TopicQueueMappingInfo cloneAsMappingInfo(TopicQueueMappingDetail mappingDetail) {
TopicQueueMappingInfo topicQueueMappingInfo = new TopicQueueMappingInfo(mappingDetail.topic, mappingDetail.totalQueues, mappingDetail.bname, mappingDetail.epoch);
topicQueueMappingInfo.currIdMap = TopicQueueMappingDetail.buildIdMap(mappingDetail, LEVEL_0);
return topicQueueMappingInfo;
}
public static boolean checkIfAsPhysical(TopicQueueMappingDetail mappingDetail, Integer globalId) {
List<LogicQueueMappingItem> mappingItems = getMappingInfo(mappingDetail, globalId);
return mappingItems == null
|| (mappingItems.size() == 1
&& mappingItems.get(0).getLogicOffset() == 0);
}
public ConcurrentMap<Integer, List<LogicQueueMappingItem>> getHostedQueues() {
return hostedQueues;
}
public void setHostedQueues(ConcurrentMap<Integer, List<LogicQueueMappingItem>> hostedQueues) {
this.hostedQueues = hostedQueues;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof TopicQueueMappingDetail)) return false;
TopicQueueMappingDetail that = (TopicQueueMappingDetail) o;
return new EqualsBuilder()
.append(hostedQueues, that.hostedQueues)
.isEquals();
}
@Override
public int hashCode() {
return new HashCodeBuilder(17, 37)
.append(hostedQueues)
.toHashCode();
}
@Override
public String toString() {
return "TopicQueueMappingDetail{" +
"hostedQueues=" + hostedQueues +
", topic='" + topic + '\'' +
", totalQueues=" + totalQueues +
", bname='" + bname + '\'' +
", epoch=" + epoch +
", dirty=" + dirty +
", currIdMap=" + currIdMap +
'}';
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingInfo.java
+161
View File
@@ -0,0 +1,161 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class TopicQueueMappingInfo extends RemotingSerializable {
public static final int LEVEL_0 = 0;
String topic; // redundant field
String scope = MixAll.METADATA_SCOPE_GLOBAL;
int totalQueues;
String bname; //identify the hosted broker name
long epoch; //important to fence the old dirty data
boolean dirty; //indicate if the data is dirty
//register to broker to construct the route
protected ConcurrentMap<Integer/*logicId*/, Integer/*physicalId*/> currIdMap = new ConcurrentHashMap<Integer, Integer>();
public TopicQueueMappingInfo() {
}
public TopicQueueMappingInfo(String topic, int totalQueues, String bname, long epoch) {
this.topic = topic;
this.totalQueues = totalQueues;
this.bname = bname;
this.epoch = epoch;
this.dirty = false;
}
public boolean isDirty() {
return dirty;
}
public void setDirty(boolean dirty) {
this.dirty = dirty;
}
public int getTotalQueues() {
return totalQueues;
}
public String getBname() {
return bname;
}
public String getTopic() {
return topic;
}
public long getEpoch() {
return epoch;
}
public void setEpoch(long epoch) {
this.epoch = epoch;
}
public void setTotalQueues(int totalQueues) {
this.totalQueues = totalQueues;
}
public ConcurrentMap<Integer, Integer> getCurrIdMap() {
return currIdMap;
}
public void setTopic(String topic) {
this.topic = topic;
}
public void setBname(String bname) {
this.bname = bname;
}
public void setCurrIdMap(ConcurrentMap<Integer, Integer> currIdMap) {
this.currIdMap = currIdMap;
}
public String getScope() {
return scope;
}
public void setScope(String scope) {
this.scope = scope;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof TopicQueueMappingInfo)) return false;
TopicQueueMappingInfo info = (TopicQueueMappingInfo) o;
if (totalQueues != info.totalQueues) return false;
if (epoch != info.epoch) return false;
if (dirty != info.dirty) return false;
if (topic != null ? !topic.equals(info.topic) : info.topic != null) return false;
if (scope != null ? !scope.equals(info.scope) : info.scope != null) return false;
if (bname != null ? !bname.equals(info.bname) : info.bname != null) return false;
return currIdMap != null ? currIdMap.equals(info.currIdMap) : info.currIdMap == null;
}
@Override
public int hashCode() {
int result = topic != null ? topic.hashCode() : 0;
result = 31 * result + (scope != null ? scope.hashCode() : 0);
result = 31 * result + totalQueues;
result = 31 * result + (bname != null ? bname.hashCode() : 0);
result = 31 * result + (int) (epoch ^ (epoch >>> 32));
result = 31 * result + (dirty ? 1 : 0);
result = 31 * result + (currIdMap != null ? currIdMap.hashCode() : 0);
return result;
}
@Override
public String toString() {
return "TopicQueueMappingInfo{" +
"topic='" + topic + '\'' +
", scope='" + scope + '\'' +
", totalQueues=" + totalQueues +
", bname='" + bname + '\'' +
", epoch=" + epoch +
", dirty=" + dirty +
", currIdMap=" + currIdMap +
'}';
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingOne.java
+88
View File
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import java.util.List;
public class TopicQueueMappingOne extends RemotingSerializable {
String topic; // redundant field
String bname; //identify the hosted broker name
Integer globalId;
List<LogicQueueMappingItem> items;
TopicQueueMappingDetail mappingDetail;
public TopicQueueMappingOne(TopicQueueMappingDetail mappingDetail, String topic, String bname, Integer globalId, List<LogicQueueMappingItem> items) {
this.mappingDetail = mappingDetail;
this.topic = topic;
this.bname = bname;
this.globalId = globalId;
this.items = items;
}
public String getTopic() {
return topic;
}
public String getBname() {
return bname;
}
public Integer getGlobalId() {
return globalId;
}
public List<LogicQueueMappingItem> getItems() {
return items;
}
public TopicQueueMappingDetail getMappingDetail() {
return mappingDetail;
}
@Override public boolean equals(Object o) {
if (this == o)
return true;
if (!(o instanceof TopicQueueMappingOne))
return false;
TopicQueueMappingOne that = (TopicQueueMappingOne) o;
if (topic != null ? !topic.equals(that.topic) : that.topic != null)
return false;
if (bname != null ? !bname.equals(that.bname) : that.bname != null)
return false;
if (globalId != null ? !globalId.equals(that.globalId) : that.globalId != null)
return false;
if (items != null ? !items.equals(that.items) : that.items != null)
return false;
return mappingDetail != null ? mappingDetail.equals(that.mappingDetail) : that.mappingDetail == null;
}
@Override public int hashCode() {
int result = topic != null ? topic.hashCode() : 0;
result = 31 * result + (bname != null ? bname.hashCode() : 0);
result = 31 * result + (globalId != null ? globalId.hashCode() : 0);
result = 31 * result + (items != null ? items.hashCode() : 0);
result = 31 * result + (mappingDetail != null ? mappingDetail.hashCode() : 0);
return result;
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicQueueMappingUtils.java
+694
View File
@@ -0,0 +1,694 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
import java.io.File;
import java.util.AbstractMap;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Random;
import java.util.Set;
public class TopicQueueMappingUtils {
public static final int DEFAULT_BLOCK_SEQ_SIZE = 10000;
public static class MappingAllocator {
Map<String, Integer> brokerNumMap = new HashMap<String, Integer>();
Map<Integer, String> idToBroker = new HashMap<Integer, String>();
//used for remapping
Map<String, Integer> brokerNumMapBeforeRemapping = null;
int currentIndex = 0;
Random random = new Random();
List<String> leastBrokers = new ArrayList<String>();
private MappingAllocator(Map<Integer, String> idToBroker, Map<String, Integer> brokerNumMap, Map<String, Integer> brokerNumMapBeforeRemapping) {
this.idToBroker.putAll(idToBroker);
this.brokerNumMap.putAll(brokerNumMap);
this.brokerNumMapBeforeRemapping = brokerNumMapBeforeRemapping;
}
private void freshState() {
int minNum = Integer.MAX_VALUE;
for (Map.Entry<String, Integer> entry : brokerNumMap.entrySet()) {
if (entry.getValue() < minNum) {
leastBrokers.clear();
leastBrokers.add(entry.getKey());
minNum = entry.getValue();
} else if (entry.getValue() == minNum) {
leastBrokers.add(entry.getKey());
}
}
//reduce the remapping
if (brokerNumMapBeforeRemapping != null
&& !brokerNumMapBeforeRemapping.isEmpty()) {
Collections.sort(leastBrokers, new Comparator<String>() {
@Override
public int compare(String o1, String o2) {
int i1 = 0, i2 = 0;
if (brokerNumMapBeforeRemapping.containsKey(o1)) {
i1 = brokerNumMapBeforeRemapping.get(o1);
}
if (brokerNumMapBeforeRemapping.containsKey(o2)) {
i2 = brokerNumMapBeforeRemapping.get(o2);
}
return i1 - i2;
}
});
} else {
//reduce the imbalance
Collections.shuffle(leastBrokers);
}
currentIndex = leastBrokers.size() - 1;
}
private String nextBroker() {
if (leastBrokers.isEmpty()) {
freshState();
}
int tmpIndex = currentIndex % leastBrokers.size();
return leastBrokers.remove(tmpIndex);
}
public Map<String, Integer> getBrokerNumMap() {
return brokerNumMap;
}
public void upToNum(int maxQueueNum) {
int currSize = idToBroker.size();
if (maxQueueNum <= currSize) {
return;
}
for (int i = currSize; i < maxQueueNum; i++) {
String nextBroker = nextBroker();
if (brokerNumMap.containsKey(nextBroker)) {
brokerNumMap.put(nextBroker, brokerNumMap.get(nextBroker) + 1);
} else {
brokerNumMap.put(nextBroker, 1);
}
idToBroker.put(i, nextBroker);
}
}
public Map<Integer, String> getIdToBroker() {
return idToBroker;
}
}
public static MappingAllocator buildMappingAllocator(Map<Integer, String> idToBroker, Map<String, Integer> brokerNumMap, Map<String, Integer> brokerNumMapBeforeRemapping) {
return new MappingAllocator(idToBroker, brokerNumMap, brokerNumMapBeforeRemapping);
}
public static Map.Entry<Long, Integer> findMaxEpochAndQueueNum(List<TopicQueueMappingDetail> mappingDetailList) {
long epoch = -1;
int queueNum = 0;
for (TopicQueueMappingDetail mappingDetail : mappingDetailList) {
if (mappingDetail.getEpoch() > epoch) {
epoch = mappingDetail.getEpoch();
}
if (mappingDetail.getTotalQueues() > queueNum) {
queueNum = mappingDetail.getTotalQueues();
}
}
return new AbstractMap.SimpleImmutableEntry<Long, Integer>(epoch, queueNum);
}
public static List<TopicQueueMappingDetail> getMappingDetailFromConfig(Collection<TopicConfigAndQueueMapping> configs) {
List<TopicQueueMappingDetail> detailList = new ArrayList<TopicQueueMappingDetail>();
for (TopicConfigAndQueueMapping configMapping : configs) {
if (configMapping.getMappingDetail() != null) {
detailList.add(configMapping.getMappingDetail());
}
}
return detailList;
}
public static Map.Entry<Long, Integer> checkNameEpochNumConsistence(String topic, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {
if (brokerConfigMap == null
|| brokerConfigMap.isEmpty()) {
return null;
}
//make sure it it not null
long maxEpoch = -1;
int maxNum = -1;
String scope = null;
for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {
String broker = entry.getKey();
TopicConfigAndQueueMapping configMapping = entry.getValue();
if (configMapping.getMappingDetail() == null) {
throw new RuntimeException("Mapping info should not be null in broker " + broker);
}
TopicQueueMappingDetail mappingDetail = configMapping.getMappingDetail();
if (!broker.equals(mappingDetail.getBname())) {
throw new RuntimeException(String.format("The broker name is not equal %s != %s ", broker, mappingDetail.getBname()));
}
if (mappingDetail.isDirty()) {
throw new RuntimeException("The mapping info is dirty in broker " + broker);
}
if (!configMapping.getTopicName().equals(mappingDetail.getTopic())) {
throw new RuntimeException("The topic name is inconsistent in broker " + broker);
}
if (topic != null
&& !topic.equals(mappingDetail.getTopic())) {
throw new RuntimeException("The topic name is not match for broker " + broker);
}
if (scope != null
&& !scope.equals(mappingDetail.getScope())) {
throw new RuntimeException(String.format("scope dose not match %s != %s in %s", mappingDetail.getScope(), scope, broker));
} else {
scope = mappingDetail.getScope();
}
if (maxEpoch != -1
&& maxEpoch != mappingDetail.getEpoch()) {
throw new RuntimeException(String.format("epoch dose not match %d != %d in %s", maxEpoch, mappingDetail.getEpoch(), mappingDetail.getBname()));
} else {
maxEpoch = mappingDetail.getEpoch();
}
if (maxNum != -1
&& maxNum != mappingDetail.getTotalQueues()) {
throw new RuntimeException(String.format("total queue number dose not match %d != %d in %s", maxNum, mappingDetail.getTotalQueues(), mappingDetail.getBname()));
} else {
maxNum = mappingDetail.getTotalQueues();
}
}
return new AbstractMap.SimpleEntry<Long, Integer>(maxEpoch, maxNum);
}
public static String getMockBrokerName(String scope) {
assert scope != null;
if (scope.equals(MixAll.METADATA_SCOPE_GLOBAL)) {
return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope.substring(2);
} else {
return MixAll.LOGICAL_QUEUE_MOCK_BROKER_PREFIX + scope;
}
}
public static void makeSureLogicQueueMappingItemImmutable(List<LogicQueueMappingItem> oldItems, List<LogicQueueMappingItem> newItems, boolean epochEqual, boolean isCLean) {
if (oldItems == null || oldItems.isEmpty()) {
return;
}
if (newItems == null || newItems.isEmpty()) {
throw new RuntimeException("The new item list is null or empty");
}
int iold = 0, inew = 0;
while (iold < oldItems.size() && inew < newItems.size()) {
LogicQueueMappingItem newItem = newItems.get(inew);
LogicQueueMappingItem oldItem = oldItems.get(iold);
if (newItem.getGen() < oldItem.getGen()) {
//the earliest item may have been deleted concurrently
inew++;
} else if (oldItem.getGen() < newItem.getGen()) {
//in the following cases, the new item-list has less items than old item-list
//1. the queue is mapped back to a broker which hold the logic queue before
//2. The earliest item is deleted by TopicQueueMappingCleanService
iold++;
} else {
assert oldItem.getBname().equals(newItem.getBname());
assert oldItem.getQueueId() == newItem.getQueueId();
assert oldItem.getStartOffset() == newItem.getStartOffset();
if (oldItem.getLogicOffset() != -1) {
assert oldItem.getLogicOffset() == newItem.getLogicOffset();
}
iold++;
inew++;
}
}
if (epochEqual) {
LogicQueueMappingItem oldLeader = oldItems.get(oldItems.size() - 1);
LogicQueueMappingItem newLeader = newItems.get(newItems.size() - 1);
if (newLeader.getGen() != oldLeader.getGen()
|| !newLeader.getBname().equals(oldLeader.getBname())
|| newLeader.getQueueId() != oldLeader.getQueueId()
|| newLeader.getStartOffset() != oldLeader.getStartOffset()) {
throw new RuntimeException("The new leader is different but epoch equal");
}
}
}
public static void checkLogicQueueMappingItemOffset(List<LogicQueueMappingItem> items) {
if (items == null
|| items.isEmpty()) {
return;
}
int lastGen = -1;
long lastOffset = -1;
for (int i = items.size() - 1; i >= 0 ; i--) {
LogicQueueMappingItem item = items.get(i);
if (item.getStartOffset() < 0
|| item.getGen() < 0
|| item.getQueueId() < 0) {
throw new RuntimeException("The field is illegal, should not be negative");
}
if (items.size() >= 2
&& i <= items.size() - 2
&& items.get(i).getLogicOffset() < 0) {
throw new RuntimeException("The non-latest item has negative logic offset");
}
if (lastGen != -1 && item.getGen() >= lastGen) {
throw new RuntimeException("The gen dose not increase monotonically");
}
if (item.getEndOffset() != -1
&& item.getEndOffset() < item.getStartOffset()) {
throw new RuntimeException("The endOffset is smaller than the start offset");
}
if (lastOffset != -1 && item.getLogicOffset() != -1) {
if (item.getLogicOffset() >= lastOffset) {
throw new RuntimeException("The base logic offset dose not increase monotonically");
}
if (item.computeMaxStaticQueueOffset() >= lastOffset) {
throw new RuntimeException("The max logic offset dose not increase monotonically");
}
}
lastGen = item.getGen();
lastOffset = item.getLogicOffset();
}
}
public static void checkIfReusePhysicalQueue(Collection<TopicQueueMappingOne> mappingOnes) {
Map<String, TopicQueueMappingOne> physicalQueueIdMap = new HashMap<String, TopicQueueMappingOne>();
for (TopicQueueMappingOne mappingOne : mappingOnes) {
for (LogicQueueMappingItem item: mappingOne.items) {
String physicalQueueId = item.getBname() + "-" + item.getQueueId();
if (physicalQueueIdMap.containsKey(physicalQueueId)) {
throw new RuntimeException(String.format("Topic %s global queue id %d and %d shared the same physical queue %s",
mappingOne.topic, mappingOne.globalId, physicalQueueIdMap.get(physicalQueueId).globalId, physicalQueueId));
} else {
physicalQueueIdMap.put(physicalQueueId, mappingOne);
}
}
}
}
public static void checkLeaderInTargetBrokers(Collection<TopicQueueMappingOne> mappingOnes, Set<String> targetBrokers) {
for (TopicQueueMappingOne mappingOne : mappingOnes) {
if (!targetBrokers.contains(mappingOne.bname)) {
throw new RuntimeException("The leader broker does not in target broker");
}
}
}
public static void checkPhysicalQueueConsistence(Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {
for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {
TopicConfigAndQueueMapping configMapping = entry.getValue();
assert configMapping != null;
assert configMapping.getMappingDetail() != null;
if (configMapping.getReadQueueNums() < configMapping.getWriteQueueNums()) {
throw new RuntimeException("Read queues is smaller than write queues");
}
for (List<LogicQueueMappingItem> items: configMapping.getMappingDetail().getHostedQueues().values()) {
for (LogicQueueMappingItem item: items) {
if (item.getStartOffset() != 0) {
throw new RuntimeException("The start offset dose not begin from 0");
}
TopicConfig topicConfig = brokerConfigMap.get(item.getBname());
if (topicConfig == null) {
throw new RuntimeException("The broker of item dose not exist");
}
if (item.getQueueId() >= topicConfig.getWriteQueueNums()) {
throw new RuntimeException("The physical queue id is overflow the write queues");
}
}
}
}
}
public static Map<Integer, TopicQueueMappingOne> checkAndBuildMappingItems(List<TopicQueueMappingDetail> mappingDetailList, boolean replace, boolean checkConsistence) {
Collections.sort(mappingDetailList, new Comparator<TopicQueueMappingDetail>() {
@Override
public int compare(TopicQueueMappingDetail o1, TopicQueueMappingDetail o2) {
return (int)(o2.getEpoch() - o1.getEpoch());
}
});
int maxNum = 0;
Map<Integer, TopicQueueMappingOne> globalIdMap = new HashMap<Integer, TopicQueueMappingOne>();
for (TopicQueueMappingDetail mappingDetail : mappingDetailList) {
if (mappingDetail.totalQueues > maxNum) {
maxNum = mappingDetail.totalQueues;
}
for (Map.Entry<Integer, List<LogicQueueMappingItem>> entry : mappingDetail.getHostedQueues().entrySet()) {
Integer globalid = entry.getKey();
checkLogicQueueMappingItemOffset(entry.getValue());
String leaderBrokerName = getLeaderBroker(entry.getValue());
if (!leaderBrokerName.equals(mappingDetail.getBname())) {
//not the leader
continue;
}
if (globalIdMap.containsKey(globalid)) {
if (!replace) {
throw new RuntimeException(String.format("The queue id is duplicated in broker %s %s", leaderBrokerName, mappingDetail.getBname()));
}
} else {
globalIdMap.put(globalid, new TopicQueueMappingOne(mappingDetail, mappingDetail.topic, mappingDetail.bname, globalid, entry.getValue()));
}
}
}
if (checkConsistence) {
if (maxNum != globalIdMap.size()) {
throw new RuntimeException(String.format("The total queue number in config dose not match the real hosted queues %d != %d", maxNum, globalIdMap.size()));
}
for (int i = 0; i < maxNum; i++) {
if (!globalIdMap.containsKey(i)) {
throw new RuntimeException(String.format("The queue number %s is not in globalIdMap", i));
}
}
}
checkIfReusePhysicalQueue(globalIdMap.values());
return globalIdMap;
}
public static String getLeaderBroker(List<LogicQueueMappingItem> items) {
return getLeaderItem(items).getBname();
}
public static LogicQueueMappingItem getLeaderItem(List<LogicQueueMappingItem> items) {
assert items.size() > 0;
return items.get(items.size() - 1);
}
public static String writeToTemp(TopicRemappingDetailWrapper wrapper, boolean after) {
String topic = wrapper.getTopic();
String data = wrapper.toJson();
String suffix = TopicRemappingDetailWrapper.SUFFIX_BEFORE;
if (after) {
suffix = TopicRemappingDetailWrapper.SUFFIX_AFTER;
}
String fileName = System.getProperty("java.io.tmpdir") + File.separator + topic + "-" + wrapper.getEpoch() + suffix;
try {
MixAll.string2File(data, fileName);
return fileName;
} catch (Exception e) {
throw new RuntimeException("write file failed " + fileName,e);
}
}
public static long blockSeqRoundUp(long offset, long blockSeqSize) {
long num = offset / blockSeqSize;
long left = offset % blockSeqSize;
if (left < blockSeqSize / 2) {
return (num + 1) * blockSeqSize;
} else {
return (num + 2) * blockSeqSize;
}
}
public static void checkTargetBrokersComplete(Set<String> targetBrokers, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {
for (String broker : brokerConfigMap.keySet()) {
if (brokerConfigMap.get(broker).getMappingDetail().getHostedQueues().isEmpty()) {
continue;
}
if (!targetBrokers.contains(broker)) {
throw new RuntimeException("The existed broker " + broker + " dose not in target brokers ");
}
}
}
public static void checkNonTargetBrokers(Set<String> targetBrokers, Set<String> nonTargetBrokers) {
for (String broker : nonTargetBrokers) {
if (targetBrokers.contains(broker)) {
throw new RuntimeException("The non-target broker exist in target broker");
}
}
}
public static TopicRemappingDetailWrapper createTopicConfigMapping(String topic, int queueNum, Set<String> targetBrokers, Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {
checkTargetBrokersComplete(targetBrokers, brokerConfigMap);
Map<Integer, TopicQueueMappingOne> globalIdMap = new HashMap<Integer, TopicQueueMappingOne>();
Map.Entry<Long, Integer> maxEpochAndNum = new AbstractMap.SimpleImmutableEntry<Long, Integer>(System.currentTimeMillis(), queueNum);
if (!brokerConfigMap.isEmpty()) {
maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);
globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(new ArrayList<TopicQueueMappingDetail>(TopicQueueMappingUtils.getMappingDetailFromConfig(brokerConfigMap.values())), false, true);
checkIfReusePhysicalQueue(globalIdMap.values());
checkPhysicalQueueConsistence(brokerConfigMap);
}
if (queueNum < globalIdMap.size()) {
throw new RuntimeException(String.format("Cannot decrease the queue num for static topic %d < %d", queueNum, globalIdMap.size()));
}
//check the queue number
if (queueNum == globalIdMap.size()) {
throw new RuntimeException("The topic queue num is equal the existed queue num, do nothing");
}
//the check is ok, now do the mapping allocation
Map<String, Integer> brokerNumMap = new HashMap<String, Integer>();
for (String broker: targetBrokers) {
brokerNumMap.put(broker, 0);
}
final Map<Integer, String> oldIdToBroker = new HashMap<Integer, String>();
for (Map.Entry<Integer, TopicQueueMappingOne> entry : globalIdMap.entrySet()) {
String leaderbroker = entry.getValue().getBname();
oldIdToBroker.put(entry.getKey(), leaderbroker);
if (!brokerNumMap.containsKey(leaderbroker)) {
brokerNumMap.put(leaderbroker, 1);
} else {
brokerNumMap.put(leaderbroker, brokerNumMap.get(leaderbroker) + 1);
}
}
TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(oldIdToBroker, brokerNumMap, null);
allocator.upToNum(queueNum);
Map<Integer, String> newIdToBroker = allocator.getIdToBroker();
//construct the topic configAndMapping
long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis());
for (Map.Entry<Integer, String> e : newIdToBroker.entrySet()) {
Integer queueId = e.getKey();
String broker = e.getValue();
if (globalIdMap.containsKey(queueId)) {
//ignore the exited
continue;
}
TopicConfigAndQueueMapping configMapping;
if (!brokerConfigMap.containsKey(broker)) {
configMapping = new TopicConfigAndQueueMapping(new TopicConfig(topic), new TopicQueueMappingDetail(topic, 0, broker, System.currentTimeMillis()));
configMapping.setWriteQueueNums(1);
configMapping.setReadQueueNums(1);
brokerConfigMap.put(broker, configMapping);
} else {
configMapping = brokerConfigMap.get(broker);
configMapping.setWriteQueueNums(configMapping.getWriteQueueNums() + 1);
configMapping.setReadQueueNums(configMapping.getReadQueueNums() + 1);
}
LogicQueueMappingItem mappingItem = new LogicQueueMappingItem(0, configMapping.getWriteQueueNums() - 1, broker, 0, 0, -1, -1, -1);
TopicQueueMappingDetail.putMappingInfo(configMapping.getMappingDetail(), queueId, new ArrayList<LogicQueueMappingItem>(Collections.singletonList(mappingItem)));
}
// set the topic config
for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {
TopicConfigAndQueueMapping configMapping = entry.getValue();
configMapping.getMappingDetail().setEpoch(newEpoch);
configMapping.getMappingDetail().setTotalQueues(queueNum);
}
//double check the config
{
TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);
globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);
checkIfReusePhysicalQueue(globalIdMap.values());
checkPhysicalQueueConsistence(brokerConfigMap);
}
return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_CREATE_OR_UPDATE, newEpoch, brokerConfigMap, new HashSet<String>(), new HashSet<String>());
}
public static TopicRemappingDetailWrapper remappingStaticTopic(String topic, Map<String, TopicConfigAndQueueMapping> brokerConfigMap, Set<String> targetBrokers) {
Map.Entry<Long, Integer> maxEpochAndNum = TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);
Map<Integer, TopicQueueMappingOne> globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);
TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);
TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());
//the check is ok, now do the mapping allocation
int maxNum = maxEpochAndNum.getValue();
Map<String, Integer> brokerNumMap = new HashMap<String, Integer>();
for (String broker: targetBrokers) {
brokerNumMap.put(broker, 0);
}
Map<String, Integer> brokerNumMapBeforeRemapping = new HashMap<String, Integer>();
for (TopicQueueMappingOne mappingOne: globalIdMap.values()) {
if (brokerNumMapBeforeRemapping.containsKey(mappingOne.bname)) {
brokerNumMapBeforeRemapping.put(mappingOne.bname, brokerNumMapBeforeRemapping.get(mappingOne.bname) + 1);
} else {
brokerNumMapBeforeRemapping.put(mappingOne.bname, 1);
}
}
TopicQueueMappingUtils.MappingAllocator allocator = TopicQueueMappingUtils.buildMappingAllocator(new HashMap<Integer, String>(), brokerNumMap, brokerNumMapBeforeRemapping);
allocator.upToNum(maxNum);
Map<String, Integer> expectedBrokerNumMap = allocator.getBrokerNumMap();
Queue<Integer> waitAssignQueues = new ArrayDeque<Integer>();
//cannot directly use the idBrokerMap from allocator, for the number of globalId maybe not in the natural order
Map<Integer, String> expectedIdToBroker = new HashMap<Integer, String>();
//the following logic will make sure that, for one broker, either "map in" or "map out"
//It can't both, map in some queues but also map out some queues.
for (Map.Entry<Integer, TopicQueueMappingOne> entry : globalIdMap.entrySet()) {
Integer queueId = entry.getKey();
TopicQueueMappingOne mappingOne = entry.getValue();
String leaderBroker = mappingOne.getBname();
if (expectedBrokerNumMap.containsKey(leaderBroker)) {
if (expectedBrokerNumMap.get(leaderBroker) > 0) {
expectedIdToBroker.put(queueId, leaderBroker);
expectedBrokerNumMap.put(leaderBroker, expectedBrokerNumMap.get(leaderBroker) - 1);
} else {
waitAssignQueues.add(queueId);
expectedBrokerNumMap.remove(leaderBroker);
}
} else {
waitAssignQueues.add(queueId);
}
}
for (Map.Entry<String, Integer> entry: expectedBrokerNumMap.entrySet()) {
String broker = entry.getKey();
Integer queueNum = entry.getValue();
for (int i = 0; i < queueNum; i++) {
Integer queueId = waitAssignQueues.poll();
assert queueId != null;
expectedIdToBroker.put(queueId, broker);
}
}
long newEpoch = Math.max(maxEpochAndNum.getKey() + 1000, System.currentTimeMillis());
//Now construct the remapping info
Set<String> brokersToMapOut = new HashSet<String>();
Set<String> brokersToMapIn = new HashSet<String>();
for (Map.Entry<Integer, String> mapEntry : expectedIdToBroker.entrySet()) {
Integer queueId = mapEntry.getKey();
String broker = mapEntry.getValue();
TopicQueueMappingOne topicQueueMappingOne = globalIdMap.get(queueId);
assert topicQueueMappingOne != null;
if (topicQueueMappingOne.getBname().equals(broker)) {
continue;
}
//remapping
final String mapInBroker = broker;
final String mapOutBroker = topicQueueMappingOne.getBname();
brokersToMapIn.add(mapInBroker);
brokersToMapOut.add(mapOutBroker);
TopicConfigAndQueueMapping mapInConfig = brokerConfigMap.get(mapInBroker);
TopicConfigAndQueueMapping mapOutConfig = brokerConfigMap.get(mapOutBroker);
if (mapInConfig == null) {
mapInConfig = new TopicConfigAndQueueMapping(new TopicConfig(topic, 0, 0), new TopicQueueMappingDetail(topic, maxNum, mapInBroker, newEpoch));
brokerConfigMap.put(mapInBroker, mapInConfig);
}
mapInConfig.setWriteQueueNums(mapInConfig.getWriteQueueNums() + 1);
mapInConfig.setReadQueueNums(mapInConfig.getReadQueueNums() + 1);
List<LogicQueueMappingItem> items = new ArrayList<LogicQueueMappingItem>(topicQueueMappingOne.getItems());
LogicQueueMappingItem last = items.get(items.size() - 1);
items.add(new LogicQueueMappingItem(last.getGen() + 1, mapInConfig.getWriteQueueNums() - 1, mapInBroker, -1, 0, -1, -1, -1));
//Use the same object
TopicQueueMappingDetail.putMappingInfo(mapInConfig.getMappingDetail(), queueId, items);
TopicQueueMappingDetail.putMappingInfo(mapOutConfig.getMappingDetail(), queueId, items);
}
for (Map.Entry<String, TopicConfigAndQueueMapping> entry : brokerConfigMap.entrySet()) {
TopicConfigAndQueueMapping configMapping = entry.getValue();
configMapping.getMappingDetail().setEpoch(newEpoch);
configMapping.getMappingDetail().setTotalQueues(maxNum);
}
//double check
{
TopicQueueMappingUtils.checkNameEpochNumConsistence(topic, brokerConfigMap);
globalIdMap = TopicQueueMappingUtils.checkAndBuildMappingItems(getMappingDetailFromConfig(brokerConfigMap.values()), false, true);
TopicQueueMappingUtils.checkPhysicalQueueConsistence(brokerConfigMap);
TopicQueueMappingUtils.checkIfReusePhysicalQueue(globalIdMap.values());
TopicQueueMappingUtils.checkLeaderInTargetBrokers(globalIdMap.values(), targetBrokers);
}
return new TopicRemappingDetailWrapper(topic, TopicRemappingDetailWrapper.TYPE_REMAPPING, newEpoch, brokerConfigMap, brokersToMapIn, brokersToMapOut);
}
public static LogicQueueMappingItem findLogicQueueMappingItem(List<LogicQueueMappingItem> mappingItems, long logicOffset, boolean ignoreNegative) {
if (mappingItems == null
|| mappingItems.isEmpty()) {
return null;
}
//Could use bi-search to polish performance
for (int i = mappingItems.size() - 1; i >= 0; i--) {
LogicQueueMappingItem item = mappingItems.get(i);
if (ignoreNegative && item.getLogicOffset() < 0) {
continue;
}
if (logicOffset >= item.getLogicOffset()) {
return item;
}
}
//if not found, maybe out of range, return the first one
for (int i = 0; i < mappingItems.size(); i++) {
LogicQueueMappingItem item = mappingItems.get(i);
if (ignoreNegative && item.getLogicOffset() < 0) {
continue;
}
}
return null;
}
public static LogicQueueMappingItem findNext(List<LogicQueueMappingItem> items, LogicQueueMappingItem currentItem, boolean ignoreNegative) {
if (items == null
|| currentItem == null) {
return null;
}
for (int i = 0; i < items.size(); i++) {
LogicQueueMappingItem item = items.get(i);
if (ignoreNegative && item.getLogicOffset() < 0) {
continue;
}
if (item.getGen() == currentItem.getGen()) {
if (i < items.size() - 1) {
item = items.get(i + 1);
if (ignoreNegative && item.getLogicOffset() < 0) {
return null;
} else {
return item;
}
} else {
return null;
}
}
}
return null;
}
public static boolean checkIfLeader(List<LogicQueueMappingItem> items, TopicQueueMappingDetail mappingDetail) {
if (items == null
|| mappingDetail == null
|| items.isEmpty()) {
return false;
}
return items.get(items.size() - 1).getBname().equals(mappingDetail.getBname());
}
}
common/src/main/java/org/apache/rocketmq/common/statictopic/TopicRemappingDetailWrapper.java
+104
View File
@@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.statictopic;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class TopicRemappingDetailWrapper extends RemotingSerializable {
public static final String TYPE_CREATE_OR_UPDATE = "CREATE_OR_UPDATE";
public static final String TYPE_REMAPPING = "REMAPPING";
public static final String SUFFIX_BEFORE = ".before";
public static final String SUFFIX_AFTER = ".after";
private String topic;
private String type;
private long epoch;
private Map<String, TopicConfigAndQueueMapping> brokerConfigMap = new HashMap<String, TopicConfigAndQueueMapping>();
private Set<String> brokerToMapIn = new HashSet<String>();
private Set<String> brokerToMapOut = new HashSet<String>();
public TopicRemappingDetailWrapper() {
}
public TopicRemappingDetailWrapper(String topic, String type, long epoch, Map<String, TopicConfigAndQueueMapping> brokerConfigMap, Set<String> brokerToMapIn, Set<String> brokerToMapOut) {
this.topic = topic;
this.type = type;
this.epoch = epoch;
this.brokerConfigMap = brokerConfigMap;
this.brokerToMapIn = brokerToMapIn;
this.brokerToMapOut = brokerToMapOut;
}
public String getTopic() {
return topic;
}
public String getType() {
return type;
}
public long getEpoch() {
return epoch;
}
public Map<String, TopicConfigAndQueueMapping> getBrokerConfigMap() {
return brokerConfigMap;
}
public Set<String> getBrokerToMapIn() {
return brokerToMapIn;
}
public Set<String> getBrokerToMapOut() {
return brokerToMapOut;
}
public void setBrokerConfigMap(Map<String, TopicConfigAndQueueMapping> brokerConfigMap) {
this.brokerConfigMap = brokerConfigMap;
}
public void setBrokerToMapIn(Set<String> brokerToMapIn) {
this.brokerToMapIn = brokerToMapIn;
}
public void setBrokerToMapOut(Set<String> brokerToMapOut) {
this.brokerToMapOut = brokerToMapOut;
}
public void setTopic(String topic) {
this.topic = topic;
}
public void setType(String type) {
this.type = type;
}
public void setEpoch(long epoch) {
this.epoch = epoch;
}
}
common/src/main/java/org/apache/rocketmq/common/sysflag/MessageSysFlag.java
-1
View File
@@ -25,7 +25,6 @@ public class MessageSysFlag {
public final static int TRANSACTION_ROLLBACK_TYPE = 0x3 << 2;
public final static int BORNHOST_V6_FLAG = 0x1 << 4;
public final static int STOREHOSTADDRESS_V6_FLAG = 0x1 << 5;
public final static int LOGICAL_QUEUE_FLAG = 0x1 << 6;
public static int getTransactionValue(final int flag) {
return flag & TRANSACTION_ROLLBACK_TYPE;
common/src/main/java/org/apache/rocketmq/common/sysflag/PullSysFlag.java
+4
View File
@@ -69,6 +69,10 @@ public class PullSysFlag {
return (sysFlag & FLAG_SUSPEND) == FLAG_SUSPEND;
}
public static int clearSuspendFlag(final int sysFlag) {
return sysFlag & (~FLAG_SUSPEND);
}
public static boolean hasSubscriptionFlag(final int sysFlag) {
return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION;
}
common/src/test/java/org/apache/rocketmq/common/ConfigManagerTest.java
+1 -4
View File
@@ -15,13 +15,10 @@ package org.apache.rocketmq.common;/*
* limitations under the License.
*/
import org.apache.rocketmq.common.ConfigManager;
import org.apache.rocketmq.common.MixAll;
import org.junit.Test;
import java.io.File;
import java.io.PrintWriter;
import java.lang.reflect.Method;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

Some files were not shown because too many files have changed in this diff Show More