diff --git a/acl/pom.xml b/acl/pom.xml index 36dae6ad04..d624934178 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java index bfe96f5303..d129c66d1c 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AclConstants.java @@ -44,6 +44,16 @@ public class AclConstants { public static final String CONFIG_TIME_STAMP = "timestamp"; + public static final String PUB = "PUB"; + + public static final String SUB = "SUB"; + + public static final String DENY = "DENY"; + + public static final String PUB_SUB = "PUB|SUB"; + + public static final String SUB_PUB = "SUB|PUB"; + public static final int ACCESS_KEY_MIN_LENGTH = 6; public static final int SECRET_KEY_MIN_LENGTH = 6; diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java index 8ceb135fbf..dadcaa304a 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/Permission.java @@ -60,14 +60,14 @@ public class Permission { return Permission.DENY; } switch (permString.trim()) { - case "PUB": + case AclConstants.PUB: return Permission.PUB; - case "SUB": + case AclConstants.SUB: return Permission.SUB; - case "PUB|SUB": - case "SUB|PUB": + case AclConstants.PUB_SUB: + case AclConstants.SUB_PUB: return Permission.PUB | Permission.SUB; - case "DENY": + case AclConstants.DENY: return Permission.DENY; default: return Permission.DENY; @@ -89,6 +89,25 @@ public class Permission { } } + public static void checkResourcePerms(List resources) { + if (resources == null || resources.isEmpty()) { + return; + } + + for (String resource : resources) { + String[] items = StringUtils.split(resource, "="); + if (items.length != 2) { + throw new AclException(String.format("Parse Resource format error for %s.\n" + + "The expected resource format is 'Res=Perm'. For example: topicA=SUB", resource)); + } + + if (!AclConstants.DENY.equals(items[1].trim()) && Permission.DENY == Permission.parsePermFromString(items[1].trim())) { + throw new AclException(String.format("Parse resource permission error for %s.\n" + + "The expected permissions are 'SUB' or 'PUB' or 'SUB|PUB' or 'PUB|SUB'.", resource)); + } + } + } + public static boolean needAdminPerm(Integer code) { return ADMIN_CODE.contains(code); } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index c182d7eb6a..078d34b10b 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -50,9 +50,9 @@ public class PlainPermissionManager { private String fileName = System.getProperty("rocketmq.acl.plain.file", DEFAULT_PLAIN_ACL_FILE); - private Map plainAccessResourceMap = new HashMap<>(); + private Map plainAccessResourceMap = new HashMap<>(); - private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); + private List globalWhiteRemoteAddressStrategy = new ArrayList<>(); private RemoteAddressStrategyFactory remoteAddressStrategyFactory = new RemoteAddressStrategyFactory(); @@ -80,7 +80,7 @@ public class PlainPermissionManager { if (globalWhiteRemoteAddressesList != null && !globalWhiteRemoteAddressesList.isEmpty()) { for (int i = 0; i < globalWhiteRemoteAddressesList.size(); i++) { globalWhiteRemoteAddressStrategy.add(remoteAddressStrategyFactory. - getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); + getRemoteAddressStrategy(globalWhiteRemoteAddressesList.getString(i))); } } @@ -89,7 +89,7 @@ public class PlainPermissionManager { List plainAccessConfigList = accounts.toJavaList(PlainAccessConfig.class); for (PlainAccessConfig plainAccessConfig : plainAccessConfigList) { PlainAccessResource plainAccessResource = buildPlainAccessResource(plainAccessConfig); - plainAccessResourceMap.put(plainAccessResource.getAccessKey(),plainAccessResource); + plainAccessResourceMap.put(plainAccessResource.getAccessKey(), plainAccessResource); } } @@ -128,12 +128,17 @@ public class PlainPermissionManager { if (plainAccessConfig == null) { log.error("Parameter value plainAccessConfig is null,Please check your parameter"); - return false; + throw new AclException("Parameter value plainAccessConfig is null, Please check your parameter"); } + Permission.checkResourcePerms(plainAccessConfig.getTopicPerms()); + Permission.checkResourcePerms(plainAccessConfig.getGroupPerms()); + Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get(AclConstants.CONFIG_ACCOUNTS); Map updateAccountMap = null; if (accounts != null) { @@ -164,8 +169,9 @@ public class PlainPermissionManager { return false; } - private Map createAclAccessConfigMap(Map existedAccountMap, PlainAccessConfig plainAccessConfig) { - + private Map createAclAccessConfigMap(Map existedAccountMap, + PlainAccessConfig plainAccessConfig) { + Map newAccountsMap = null; if (existedAccountMap == null) { newAccountsMap = new LinkedHashMap(); @@ -176,8 +182,8 @@ public class PlainPermissionManager { if (StringUtils.isEmpty(plainAccessConfig.getAccessKey()) || plainAccessConfig.getAccessKey().length() <= AclConstants.ACCESS_KEY_MIN_LENGTH) { throw new AclException(String.format( - "The accessKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey())); + "The accessKey=%s cannot be null and length should longer than 6", + plainAccessConfig.getAccessKey())); } newAccountsMap.put(AclConstants.CONFIG_ACCESS_KEY, plainAccessConfig.getAccessKey()); @@ -218,8 +224,10 @@ public class PlainPermissionManager { } Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - Map.class); - + Map.class); + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List> accounts = (List>) aclAccessConfigMap.get("accounts"); if (accounts != null) { Iterator> itemIterator = accounts.iterator(); @@ -251,7 +259,9 @@ public class PlainPermissionManager { Map aclAccessConfigMap = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, Map.class); - + if (aclAccessConfigMap == null || aclAccessConfigMap.isEmpty()) { + throw new AclException(String.format("the %s file is not found or empty", fileHome + File.separator + fileName)); + } List globalWhiteRemoteAddrList = (List) aclAccessConfigMap.get(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS); if (globalWhiteRemoteAddrList != null) { @@ -259,7 +269,7 @@ public class PlainPermissionManager { globalWhiteRemoteAddrList.addAll(globalWhiteAddrsList); // Update globalWhiteRemoteAddr element in memeory map firstly - aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS,globalWhiteRemoteAddrList); + aclAccessConfigMap.put(AclConstants.CONFIG_GLOBAL_WHITE_ADDRS, globalWhiteRemoteAddrList); if (AclUtils.writeDataObject(fileHome + File.separator + fileName, updateAclConfigFileVersion(aclAccessConfigMap))) { return true; } @@ -275,7 +285,7 @@ public class PlainPermissionManager { List configs = new ArrayList<>(); List whiteAddrs = new ArrayList<>(); JSONObject plainAclConfData = AclUtils.getYamlDataObject(fileHome + File.separator + fileName, - JSONObject.class); + JSONObject.class); if (plainAclConfData == null || plainAclConfData.isEmpty()) { throw new AclException(String.format("%s file is not data", fileHome + File.separator + fileName)); } @@ -359,7 +369,7 @@ public class PlainPermissionManager { || plainAccessConfig.getSecretKey().length() <= AclConstants.SECRET_KEY_MIN_LENGTH) { throw new AclException(String.format( "The accessKey=%s and secretKey=%s cannot be null and length should longer than 6", - plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); + plainAccessConfig.getAccessKey(), plainAccessConfig.getSecretKey())); } PlainAccessResource plainAccessResource = new PlainAccessResource(); plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); @@ -375,7 +385,7 @@ public class PlainPermissionManager { Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory. - getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); + getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); return plainAccessResource; } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java index 0c3548eeb5..6ec90ee3fa 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/RemoteAddressStrategyFactory.java @@ -193,7 +193,7 @@ public class RemoteAddressStrategyFactory { throw new AclException(String.format("RangeRemoteAddressStrategy netaddress examine scope Exception start is %s , end is %s", start, end)); } } - return this.end > 0 ? true : false; + return this.end > 0; } private boolean ipv6Analysis(String[] strArray, int index) { diff --git a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java index 253b5b241e..c824065f7f 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/common/PermissionTest.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.acl.common; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -165,4 +166,27 @@ public class PermissionTest { aclException.setStatus("netaddress examine scope Exception netaddress"); Assert.assertEquals(aclException.getStatus(),"netaddress examine scope Exception netaddress"); } + + @Test + public void checkResourcePermsNormalTest() { + Permission.checkResourcePerms(null); + Permission.checkResourcePerms(new ArrayList<>()); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB")); + Permission.checkResourcePerms(Arrays.asList("topicA=PUB", "topicB=SUB", "topicC=PUB|SUB")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest1() { + Permission.checkResourcePerms(Arrays.asList("topicA")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest2() { + Permission.checkResourcePerms(Arrays.asList("topicA=")); + } + + @Test(expected = AclException.class) + public void checkResourcePermsExceptionTest3() { + Permission.checkResourcePerms(Arrays.asList("topicA=DENY1")); + } } diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java index 645e522763..c3d41919a7 100644 --- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java @@ -546,6 +546,26 @@ public class PlainAccessValidatorTest { Assert.assertEquals(plainAccessValidator.updateAccessConfig(plainAccessConfig), false); } + @Test(expected = AclException.class) + public void createAndUpdateAccessAclYamlConfigExceptionTest() { + System.setProperty("rocketmq.home.dir", "src/test/resources"); + System.setProperty("rocketmq.acl.plain.file", "/conf/plain_acl_update_create.yml"); + + PlainAccessConfig plainAccessConfig = new PlainAccessConfig(); + plainAccessConfig.setAccessKey("RocketMQ33"); + plainAccessConfig.setSecretKey("123456789111"); + List topicPerms = new ArrayList(); + topicPerms.add("topicB=PUB"); + plainAccessConfig.setTopicPerms(topicPerms); + List groupPerms = new ArrayList(); + groupPerms.add("groupC=DENY1"); + plainAccessConfig.setGroupPerms(groupPerms); + + PlainAccessValidator plainAccessValidator = new PlainAccessValidator(); + // Create element in the acl access yaml config file + plainAccessValidator.updateAccessConfig(plainAccessConfig); + } + @Test public void updateGlobalWhiteAddrsNormalTest() { System.setProperty("rocketmq.home.dir", "src/test/resources"); diff --git a/broker/pom.xml b/broker/pom.xml index 64ae7d9955..8453452261 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java index f4c30d5d64..877ddd83b4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/loadbalance/AssignmentManager.java @@ -90,7 +90,7 @@ public class AssignmentManager { log.error("ScheduledTask: failed to pull TopicRouteData from NameServer", e); } } - }, 13000, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS); + }, 200, this.brokerController.getBrokerConfig().getLoadBalancePollNameServerInterval(), TimeUnit.MILLISECONDS); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java index a74b6d66ce..a89bace193 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/mqtrace/SendMessageHook.java @@ -17,9 +17,9 @@ package org.apache.rocketmq.broker.mqtrace; public interface SendMessageHook { - public String hookName(); + String hookName(); - public void sendMessageBefore(final SendMessageContext context); + void sendMessageBefore(final SendMessageContext context); - public void sendMessageAfter(final SendMessageContext context); + void sendMessageAfter(final SendMessageContext context); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index d8f53e4542..724cf54c81 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -218,6 +218,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement String originMsgId = MessageAccessor.getOriginMessageId(msgExt); MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() : originMsgId); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + CompletableFuture putMessageResult = this.brokerController.getMessageStore().asyncPutMessage(msgInner); return putMessageResult.thenApply((r) -> { if (r != null) { diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java index 68d58ef440..9ea1eeee31 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerOuterAPITest.java @@ -124,7 +124,7 @@ public class BrokerOuterAPITest { boolean success = Iterables.any(booleanList, new Predicate() { public boolean apply(Boolean input) { - return input ? true : false; + return input; } }); diff --git a/client/pom.xml b/client/pom.xml index 164082c9b2..95ef4617dd 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 @@ -47,6 +47,10 @@ org.apache.commons commons-lang3 + + commons-codec + commons-codec + io.opentracing opentracing-api diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java index c0b04caea1..891c17e3ba 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java @@ -30,9 +30,8 @@ public class ThreadLocalIndex { this.threadLocalIndex.set(index); } - index = Math.abs(index + 1); - this.threadLocalIndex.set(index); - return index; + this.threadLocalIndex.set(++index); + return Math.abs(index); } @Override diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 393bda928e..c54399aa5e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -23,6 +23,10 @@ import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAverage import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; +import org.apache.rocketmq.client.trace.TraceDispatcher; +import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; @@ -30,10 +34,13 @@ import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { + private final InternalLogger log = ClientLogger.getLog(); + private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** @@ -153,6 +160,21 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() - (1000 * 60 * 30)); + /** + * Interface of asynchronous transfer data + */ + private TraceDispatcher traceDispatcher = null; + + /** + * The flag for message trace + */ + private boolean enableMsgTrace = false; + + /** + * The name value of message trace topic.If you don't config,you can use the default trace topic name. + */ + private String customizedTraceTopic; + /** * Default constructor. */ @@ -202,13 +224,24 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void start() throws MQClientException { + setTraceDispatcher(); setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup)); this.defaultLitePullConsumerImpl.start(); + if (null != traceDispatcher) { + try { + traceDispatcher.start(this.getNamesrvAddr(), this.getAccessChannel()); + } catch (MQClientException e) { + log.warn("trace dispatcher start failed ", e); + } + } } @Override public void shutdown() { this.defaultLitePullConsumerImpl.shutdown(); + if (null != traceDispatcher) { + traceDispatcher.shutdown(); + } } @Override @@ -490,4 +523,36 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setConsumeTimestamp(String consumeTimestamp) { this.consumeTimestamp = consumeTimestamp; } + + public TraceDispatcher getTraceDispatcher() { + return traceDispatcher; + } + + public void setCustomizedTraceTopic(String customizedTraceTopic) { + this.customizedTraceTopic = customizedTraceTopic; + } + + private void setTraceDispatcher() { + if (isEnableMsgTrace()) { + try { + this.traceDispatcher = new AsyncTraceDispatcher(consumerGroup, TraceDispatcher.Type.CONSUME, customizedTraceTopic, null); + this.defaultLitePullConsumerImpl.registerConsumeMessageHook( + new ConsumeMessageTraceHookImpl(traceDispatcher)); + } catch (Throwable e) { + log.error("system mqtrace hook init failed ,maybe can't send msg trace data"); + } + } + } + + public String getCustomizedTraceTopic() { + return customizedTraceTopic; + } + + public boolean isEnableMsgTrace() { + return enableMsgTrace; + } + + public void setEnableMsgTrace(boolean enableMsgTrace) { + this.enableMsgTrace = enableMsgTrace; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index d32dd158e3..f32215ab83 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -281,7 +281,7 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume private TraceDispatcher traceDispatcher = null; // force to use client rebalance - private boolean clientRebalance = false; + private boolean clientRebalance = true; /** * Default constructor. diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java index 2128ffd0dd..8884e4adfe 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java @@ -268,17 +268,23 @@ public class MQAdminImpl { messageId.getOffset(), timeoutMillis); } - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return queryMessage(topic, key, maxNum, begin, end, false); } + public QueryResult queryMessageByUniqKey(String topic, String uniqKey, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return queryMessage(topic, uniqKey, maxNum, begin, end, true); + } + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws InterruptedException, MQClientException { - QueryResult qr = this.queryMessage(topic, uniqKey, 32, - MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE, true); + QueryResult qr = queryMessageByUniqKey(topic, uniqKey, 32, + MessageClientIDSetter.getNearlyTimeFromID(uniqKey).getTime() - 1000, Long.MAX_VALUE); if (qr != null && qr.getMessageList() != null && qr.getMessageList().size() > 0) { return qr.getMessageList().get(0); } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index c64c240f2f..2e73f1a517 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -41,12 +41,15 @@ import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; @@ -115,6 +118,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { */ private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + private static final long PULL_TIME_DELAY_MILLS_ON_EXCEPTION = 3 * 1000; + private DefaultLitePullConsumer defaultLitePullConsumer; private final ConcurrentMap taskTable = @@ -142,6 +147,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + private final ArrayList consumeMessageHookList = new ArrayList<>(); + // only for test purpose, will be modified by reflection in unit test. @SuppressWarnings("FieldMayBeFinal") private static boolean doNotUpdateTopicSubscribeInfoWhenSubscriptionChanged = false; @@ -161,6 +168,35 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookBefore exception", hook.hookName(), e); + } + } + } + } + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable e) { + log.error("consumeMessageHook {} executeHookAfter exception", hook.hookName(), e); + } + } + } + } + private void checkServiceState() { if (this.serviceState != ServiceState.RUNNING) throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); @@ -632,9 +668,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long fetchConsumeOffset(MessageQueue messageQueue) { + private long fetchConsumeOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); - long offset = this.rebalanceImpl.computePullFromWhere(messageQueue); + long offset = this.rebalanceImpl.computePullFromWhereWithException(messageQueue); return offset; } @@ -658,7 +694,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long nextPullOffset(MessageQueue messageQueue) { + private long nextPullOffset(MessageQueue messageQueue) throws MQClientException { long offset = -1; long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); if (seekOffset != -1) { @@ -745,7 +781,15 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return; } - long offset = nextPullOffset(messageQueue); + long offset = 0L; + try { + offset = nextPullOffset(messageQueue); + } catch (MQClientException e) { + log.error("Failed to get next pull offset", e); + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_ON_EXCEPTION, TimeUnit.MILLISECONDS); + return; + } + if (this.isCancelled() || processQueue.isDropped()) { return; } @@ -759,7 +803,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { String topic = this.messageQueue.getTopic(); subscriptionData = FilterAPI.buildSubscriptionData(topic, SubscriptionData.SUB_ALL); } - + PullResult pullResult = pull(messageQueue, subscriptionData, offset, defaultLitePullConsumer.getPullBatchSize()); if (this.isCancelled() || processQueue.isDropped()) { return; @@ -854,6 +898,18 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + if (!this.consumeMessageHookList.isEmpty()) { + ConsumeMessageContext consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); + consumeMessageContext.setConsumerGroup(this.groupName()); + consumeMessageContext.setMq(mq); + consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); + consumeMessageContext.setSuccess(false); + this.executeHookBefore(consumeMessageContext); + consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + consumeMessageContext.setSuccess(true); + this.executeHookAfter(consumeMessageContext); + } return pullResult; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 64126b07ce..b478cb1093 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -293,8 +293,15 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } } else { if (processQueue.isLocked()) { - if (!pullRequest.isLockedFirst()) { - final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); + if (!pullRequest.isPreviouslyLocked()) { + long offset = -1L; + try { + offset = this.rebalanceImpl.computePullFromWhereWithException(pullRequest.getMessageQueue()); + } catch (MQClientException e) { + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); + log.error("Failed to compute pull offset, pullResult: {}", pullRequest, e); + return; + } boolean brokerBusy = offset < pullRequest.getNextOffset(); log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy); @@ -303,7 +310,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { pullRequest, offset); } - pullRequest.setLockedFirst(true); + pullRequest.setPreviouslyLocked(true); pullRequest.setNextOffset(offset); } } else { @@ -577,8 +584,8 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { invisibleTime = 60000; } this.pullAPIWrapper.popAsync(popRequest.getMessageQueue(), invisibleTime, this.defaultMQPushConsumer.getPopBatchNums(), - popRequest.getConsumerGroup(), BROKER_SUSPEND_MAX_TIME_MILLIS, popCallback, true, popRequest.getInitMode(), - false, subscriptionData.getExpressionType(), subscriptionData.getSubString()); + popRequest.getConsumerGroup(), BROKER_SUSPEND_MAX_TIME_MILLIS, popCallback, true, popRequest.getInitMode(), + false, subscriptionData.getExpressionType(), subscriptionData.getSubString()); } catch (Exception e) { log.error("popAsync exception", e); this.executePopPullRequestLater(popRequest, pullTimeDelayMillsWhenException); @@ -590,7 +597,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { List msgFoundList = popResult.getMsgFoundList(); List msgListFilterAgain = msgFoundList; if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode() - && popResult.getMsgFoundList().size() > 0) { + && popResult.getMsgFoundList().size() > 0) { msgListFilterAgain = new ArrayList(popResult.getMsgFoundList().size()); for (MessageExt msg : popResult.getMsgFoundList()) { if (msg.getTags() != null) { @@ -773,12 +780,12 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } void changePopInvisibleTimeAsync(String topic, String consumerGroup, String extraInfo, long invisibleTime, AckCallback callback) - throws MQClientException, RemotingException, InterruptedException, MQBrokerException { + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); String brokerName = ExtraInfoUtil.getBrokerName(extraInfoStrs); int queueId = ExtraInfoUtil.getQueueId(extraInfoStrs); FindBrokerResult - findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); if (null == findBrokerResult) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); findBrokerResult = this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); @@ -1094,19 +1101,19 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { // popInvisibleTime if (this.defaultMQPushConsumer.getPopInvisibleTime() < MIN_POP_INVISIBLE_TIME - || this.defaultMQPushConsumer.getPopInvisibleTime() > MAX_POP_INVISIBLE_TIME) { + || this.defaultMQPushConsumer.getPopInvisibleTime() > MAX_POP_INVISIBLE_TIME) { throw new MQClientException( - "popInvisibleTime Out of range [" + MIN_POP_INVISIBLE_TIME + ", " + MAX_POP_INVISIBLE_TIME + "]" - + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), - null); + "popInvisibleTime Out of range [" + MIN_POP_INVISIBLE_TIME + ", " + MAX_POP_INVISIBLE_TIME + "]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); } // popBatchNums if (this.defaultMQPushConsumer.getPopBatchNums() <= 0 || this.defaultMQPushConsumer.getPopBatchNums() > 32) { throw new MQClientException( - "popBatchNums Out of range [1, 32]" - + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), - null); + "popBatchNums Out of range [1, 32]" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java index 71d1fdbe92..b90192b992 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullRequest.java @@ -24,14 +24,14 @@ public class PullRequest implements MessageRequest { private MessageQueue messageQueue; private ProcessQueue processQueue; private long nextOffset; - private boolean lockedFirst = false; + private boolean previouslyLocked = false; - public boolean isLockedFirst() { - return lockedFirst; + public boolean isPreviouslyLocked() { + return previouslyLocked; } - public void setLockedFirst(boolean lockedFirst) { - this.lockedFirst = lockedFirst; + public void setPreviouslyLocked(boolean previouslyLocked) { + this.previouslyLocked = previouslyLocked; } public String getConsumerGroup() { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 7e78c9b57f..7a457c152b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -28,6 +28,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; @@ -391,17 +392,7 @@ public abstract class RebalanceImpl { } private boolean getRebalanceResultFromBroker(final String topic, final boolean isOrder) { - String strategyName; - switch (messageModel) { - case BROADCASTING: - strategyName = null; - break; - case CLUSTERING: - strategyName = this.allocateMessageQueueStrategy.getName(); - break; - default: - return true; - } + String strategyName = this.allocateMessageQueueStrategy.getName(); Set messageQueueAssignments; try { messageQueueAssignments = this.mQClientFactory.queryAssignment(topic, consumerGroup, @@ -422,9 +413,6 @@ public abstract class RebalanceImpl { } } Set mqAll = null; - if (messageModel == MessageModel.BROADCASTING) { - mqAll = mqSet; - } boolean changed = this.updateMessageQueueAssignment(topic, messageQueueAssignments, isOrder); if (changed) { log.info("broker rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, clientId={}, assignmentSet={}", @@ -705,7 +693,14 @@ public abstract class RebalanceImpl { this.removeDirtyOffset(mq); ProcessQueue pq = createProcessQueue(); pq.setLocked(true); - long nextOffset = this.computePullFromWhere(mq); + long nextOffset = -1L; + try { + nextOffset = this.computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.info("doRebalance, {}, compute offset failed, {}", consumerGroup, mq); + continue; + } + if (nextOffset >= 0) { ProcessQueue pre = this.processQueueTable.putIfAbsent(mq, pq); if (pre != null) { @@ -774,8 +769,17 @@ public abstract class RebalanceImpl { public abstract void removeDirtyOffset(final MessageQueue mq); + /** + * When the network is unstable, using this interface may return wrong offset. + * It is recommended to use computePullFromWhereWithException instead. + * @param mq + * @return offset + */ + @Deprecated public abstract long computePullFromWhere(final MessageQueue mq); + public abstract long computePullFromWhereWithException(final MessageQueue mq) throws MQClientException; + public abstract int getConsumeInitMode(); public abstract void dispatchPullRequest(final List pullRequestList, final long delay); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java index c8f5cf1b1f..4d347113c7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -74,8 +74,20 @@ public class RebalanceLitePullImpl extends RebalanceImpl { this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { ConsumeFromWhere consumeFromWhere = litePullConsumerImpl.getDefaultLitePullConsumer().getConsumeFromWhere(); long result = -1; switch (consumeFromWhere) { @@ -118,7 +130,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -126,7 +139,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java index a68fb04eef..e1c67926a9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Set; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; @@ -68,11 +69,17 @@ public class RebalancePullImpl extends RebalanceImpl { this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { return 0; } + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { + return 0; + } + @Override public int getConsumeInitMode() { throw new UnsupportedOperationException("no initMode for Pull"); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java index 88c7c1e477..09d1521654 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -115,7 +115,7 @@ public class RebalancePushImpl extends RebalanceImpl { @Override public boolean clientRebalance(String topic) { // POPTODO order pop consume not implement yet - return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly(); + return defaultMQPushConsumerImpl.getDefaultMQPushConsumer().isClientRebalance() || defaultMQPushConsumerImpl.isConsumeOrderly() || MessageModel.BROADCASTING.equals(messageModel); } public boolean removeUnnecessaryPopMessageQueue(final MessageQueue mq, final PopProcessQueue pq) { @@ -149,8 +149,20 @@ public class RebalancePushImpl extends RebalanceImpl { this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); } + @Deprecated @Override public long computePullFromWhere(MessageQueue mq) { + long result = -1L; + try { + result = computePullFromWhereWithException(mq); + } catch (MQClientException e) { + log.warn("Compute consume offset exception, mq={}", mq); + } + return result; + } + + @Override + public long computePullFromWhereWithException(MessageQueue mq) throws MQClientException { long result = -1; final ConsumeFromWhere consumeFromWhere = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere(); final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore(); @@ -171,7 +183,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { @@ -199,7 +212,8 @@ public class RebalancePushImpl extends RebalanceImpl { try { result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } else { try { @@ -207,7 +221,8 @@ public class RebalancePushImpl extends RebalanceImpl { UtilAll.YYYYMMDDHHMMSS).getTime(); result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } catch (MQClientException e) { - result = -1; + log.warn("Compute consume offset from last offset exception, mq={}, exception={}", mq, e); + throw e; } } } else { diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java index b2b06452f6..b65ba46a39 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceDataEncoder.java @@ -85,7 +85,6 @@ public class TraceDataEncoder { bean.setMsgId(line[5]); bean.setRetryTimes(Integer.parseInt(line[6])); bean.setKeys(line[7]); - bean.setClientHost(line[8]); subBeforeContext.setTraceBeans(new ArrayList(1)); subBeforeContext.getTraceBeans().add(bean); resList.add(subBeforeContext); @@ -123,10 +122,9 @@ public class TraceDataEncoder { bean.setKeys(line[7]); bean.setStoreHost(line[8]); bean.setMsgType(MessageType.values()[Integer.parseInt(line[9])]); - bean.setClientHost(line[10]); - bean.setTransactionId(line[11]); - bean.setTransactionState(LocalTransactionState.valueOf(line[12])); - bean.setFromTransactionCheck(Boolean.parseBoolean(line[13])); + bean.setTransactionId(line[10]); + bean.setTransactionState(LocalTransactionState.valueOf(line[11])); + bean.setFromTransactionCheck(Boolean.parseBoolean(line[12])); endTransactionContext.setTraceBeans(new ArrayList(1)); endTransactionContext.getTraceBeans().add(bean); @@ -166,8 +164,7 @@ public class TraceDataEncoder { .append(ctx.getCostTime()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getOffsetMsgId()).append(TraceConstants.CONTENT_SPLITOR)// - .append(ctx.isSuccess()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.FIELD_SPLITOR); + .append(ctx.isSuccess()).append(TraceConstants.FIELD_SPLITOR);// } break; case SubBefore: { @@ -179,8 +176,7 @@ public class TraceDataEncoder { .append(ctx.getRequestId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getRetryTimes()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.FIELD_SPLITOR);// + .append(bean.getKeys()).append(TraceConstants.FIELD_SPLITOR);// } } break; @@ -198,6 +194,7 @@ public class TraceDataEncoder { } } + break; case EndTransaction: { TraceBean bean = ctx.getTraceBeans().get(0); sb.append(ctx.getTraceType()).append(TraceConstants.CONTENT_SPLITOR)// @@ -210,7 +207,6 @@ public class TraceDataEncoder { .append(bean.getKeys()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getStoreHost()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getMsgType().ordinal()).append(TraceConstants.CONTENT_SPLITOR)// - .append(bean.getClientHost()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getTransactionId()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.getTransactionState().name()).append(TraceConstants.CONTENT_SPLITOR)// .append(bean.isFromTransactionCheck()).append(TraceConstants.FIELD_SPLITOR); diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java index 14fc360f11..e78d37ab2c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/TraceView.java @@ -17,10 +17,10 @@ package org.apache.rocketmq.client.trace; - - import java.util.ArrayList; import java.util.List; +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; public class TraceView { @@ -38,8 +38,9 @@ public class TraceView { private String groupName; private String status; - public static List decodeFromTraceTransData(String key, String messageBody) { + public static List decodeFromTraceTransData(String key, MessageExt messageExt) { List messageTraceViewList = new ArrayList(); + String messageBody = new String(messageExt.getBody(), Charsets.UTF_8); if (messageBody == null || messageBody.length() <= 0) { return messageTraceViewList; } @@ -56,8 +57,7 @@ public class TraceView { messageTraceView.setGroupName(context.getGroupName()); if (context.isSuccess()) { messageTraceView.setStatus("success"); - } - else { + } else { messageTraceView.setStatus("failed"); } messageTraceView.setKeys(traceBean.getKeys()); @@ -68,7 +68,7 @@ public class TraceView { messageTraceView.setOffSetMsgId(traceBean.getOffsetMsgId()); messageTraceView.setTimeStamp(context.getTimeStamp()); messageTraceView.setStoreHost(traceBean.getStoreHost()); - messageTraceView.setClientHost(traceBean.getClientHost()); + messageTraceView.setClientHost(messageExt.getBornHostString()); messageTraceViewList.add(messageTraceView); } return messageTraceViewList; diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java index 4f6f916ea9..bce613987f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/ConsumeMessageTraceHookImpl.java @@ -16,10 +16,10 @@ */ package org.apache.rocketmq.client.trace.hook; +import java.util.Map; import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; -import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceContext; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.TraceBean; @@ -74,7 +74,6 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { traceBean.setStoreTime(msg.getStoreTimestamp());// traceBean.setBodyLength(msg.getStoreSize());// traceBean.setRetryTimes(msg.getReconsumeTimes());// - traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostConsumer().getmQClientFactory().getClientId()); traceContext.setRegionId(regionId);// beans.add(traceBean); } @@ -93,7 +92,7 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { TraceContext subBeforeContext = (TraceContext) context.getMqTraceContext(); if (subBeforeContext.getTraceBeans() == null || subBeforeContext.getTraceBeans().size() < 1) { - // If subbefore bean is null ,skip it + // If subBefore bean is null ,skip it return; } TraceContext subAfterContext = new TraceContext(); @@ -103,13 +102,16 @@ public class ConsumeMessageTraceHookImpl implements ConsumeMessageHook { subAfterContext.setRequestId(subBeforeContext.getRequestId());// subAfterContext.setSuccess(context.isSuccess());// - // Caculate the cost time for processing messages + // Calculate the cost time for processing messages int costTime = (int) ((System.currentTimeMillis() - subBeforeContext.getTimeStamp()) / context.getMsgList().size()); subAfterContext.setCostTime(costTime);// subAfterContext.setTraceBeans(subBeforeContext.getTraceBeans()); - String contextType = context.getProps().get(MixAll.CONSUME_CONTEXT_TYPE); - if (contextType != null) { - subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + Map props = context.getProps(); + if (props != null) { + String contextType = props.get(MixAll.CONSUME_CONTEXT_TYPE); + if (contextType != null) { + subAfterContext.setContextCode(ConsumeReturnType.valueOf(contextType).ordinal()); + } } localDispatcher.append(subAfterContext); } diff --git a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java index 4feb276284..80c7babdaa 100644 --- a/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/trace/hook/SendMessageTraceHookImpl.java @@ -60,7 +60,6 @@ public class SendMessageTraceHookImpl implements SendMessageHook { traceBean.setStoreHost(context.getBrokerAddr()); traceBean.setBodyLength(context.getMessage().getBody().length); traceBean.setMsgType(context.getMsgType()); - traceBean.setClientHost(((AsyncTraceDispatcher)localDispatcher).getHostProducer().getmQClientFactory().getClientId()); tuxeContext.getTraceBeans().add(traceBean); } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index 4190e286a2..7c3c501e05 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -57,6 +57,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; @@ -73,6 +74,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; @@ -80,7 +82,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -@RunWith(MockitoJUnitRunner.class) +@RunWith(MockitoJUnitRunner.Silent.class) public class DefaultMQPushConsumerTest { private String consumerGroup; private String topic = "FooBar"; @@ -105,6 +107,7 @@ public class DefaultMQPushConsumerTest { pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.setNamesrvAddr("127.0.0.1:9876"); pushConsumer.setPullInterval(60 * 1000); + pushConsumer.setClientRebalance(false); pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @Override @@ -121,8 +124,10 @@ public class DefaultMQPushConsumerTest { mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + doReturn(null).when(mQClientFactory).queryAssignment(anyString(), anyString(), anyString(), any(MessageModel.class), anyInt()); rebalanceImpl = spy(pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl()); + doReturn(123L).when(rebalanceImpl).computePullFromWhereWithException(any(MessageQueue.class)); Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); field.setAccessible(true); field.set(pushConsumerImpl, rebalanceImpl); @@ -174,7 +179,6 @@ public class DefaultMQPushConsumerTest { Set messageQueueSet = new HashSet(); messageQueueSet.add(createPullRequest().getMessageQueue()); pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); - doReturn(123L).when(rebalanceImpl).computePullFromWhere(any(MessageQueue.class)); } @After @@ -286,7 +290,7 @@ public class DefaultMQPushConsumerTest { pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService(new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), new MessageListenerConcurrently() { @Override public ConsumeConcurrentlyStatus consumeMessage(List msgs, - ConsumeConcurrentlyContext context) { + ConsumeConcurrentlyContext context) { assertThat(msgs.get(0).getBody()).isEqualTo(msgBody); countDownLatch.countDown(); try { @@ -345,4 +349,21 @@ public class DefaultMQPushConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + + @Test + public void testPullMessage_ExceptionOccursWhenComputePullFromWhere() throws MQClientException { + final CountDownLatch countDownLatch = new CountDownLatch(1); + final MessageExt[] messageExts = new MessageExt[1]; + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeMessageService( + new ConsumeMessageConcurrentlyService(pushConsumer.getDefaultMQPushConsumerImpl(), + (msgs, context) -> { + messageExts[0] = msgs.get(0); + return null; + })); + + pushConsumer.getDefaultMQPushConsumerImpl().setConsumeOrderly(true); + PullMessageService pullMessageService = mQClientFactory.getPullMessageService(); + pullMessageService.executePullRequestImmediately(createPullRequest()); + assertThat(messageExts[0]).isNull(); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java index b7a42e1533..17b48eba51 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/RebalancePushImplTest.java @@ -32,6 +32,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -58,6 +59,11 @@ public class RebalancePushImplTest { private String topic = "TopicA"; private final String brokerName = "BrokerA"; + @Before + public void before() { + defaultMQPushConsumer.getDefaultMQPushConsumer().setClientRebalance(false); + } + @Test public void testMessageQueueChanged_CountThreshold() { RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING, diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java index c173b8ef79..ecf72ae44c 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQConsumerWithOpenTracingTest.java @@ -21,7 +21,6 @@ import io.opentracing.mock.MockSpan; import io.opentracing.mock.MockTracer; import io.opentracing.tag.Tags; import java.io.ByteArrayOutputStream; -import java.lang.reflect.Field; import java.net.InetSocketAddress; import java.util.Collections; import java.util.HashSet; @@ -75,6 +74,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.waitAtMost; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; @@ -105,6 +105,27 @@ public class DefaultMQConsumerWithOpenTracingTest { factoryTable.forEach((s, instance) -> instance.shutdown()); factoryTable.clear(); + when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public PullResult answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[]{'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); + return pullResult; + } + }); + + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.getDefaultMQPushConsumerImpl().registerConsumeMessageHook( @@ -128,58 +149,20 @@ public class DefaultMQConsumerWithOpenTracingTest { // suppress updateTopicRouteInfoFromNameServer pushConsumer.changeInstanceNameToPID(); - mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true))); + mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(pushConsumer, (RPCHook) FieldUtils.readDeclaredField(pushConsumerImpl, "rpcHook", true)); + FieldUtils.writeDeclaredField(mQClientFactory, "mQClientAPIImpl", mQClientAPIImpl, true); + mQClientFactory = spy(mQClientFactory); factoryTable.put(pushConsumer.buildMQClientId(), mQClientFactory); doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); - rebalancePushImpl = spy(new RebalancePushImpl(pushConsumer.getDefaultMQPushConsumerImpl())); - Field field = DefaultMQPushConsumerImpl.class.getDeclaredField("rebalanceImpl"); - field.setAccessible(true); - field.set(pushConsumerImpl, rebalancePushImpl); - pushConsumer.subscribe(topic, "*"); - - pushConsumer.start(); - - field = DefaultMQPushConsumerImpl.class.getDeclaredField("mQClientFactory"); - field.setAccessible(true); - field.set(pushConsumerImpl, mQClientFactory); - - field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); - field.setAccessible(true); - field.set(mQClientFactory, mQClientAPIImpl); - - pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); - field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); - field.setAccessible(true); - field.set(pushConsumerImpl, pullAPIWrapper); - - pushConsumer.getDefaultMQPushConsumerImpl().getRebalanceImpl().setmQClientFactory(mQClientFactory); - mQClientFactory.registerConsumer(consumerGroup, pushConsumerImpl); - - when(mQClientAPIImpl.pullMessage(anyString(), any(PullMessageRequestHeader.class), - anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) - .thenAnswer(new Answer() { - @Override - public PullResult answer(InvocationOnMock mock) throws Throwable { - PullMessageRequestHeader requestHeader = mock.getArgument(1); - MessageClientExt messageClientExt = new MessageClientExt(); - messageClientExt.setTopic(topic); - messageClientExt.setQueueId(0); - messageClientExt.setMsgId("123"); - messageClientExt.setBody(new byte[]{'a'}); - messageClientExt.setOffsetMsgId("234"); - messageClientExt.setBornHost(new InetSocketAddress(8080)); - messageClientExt.setStoreHost(new InetSocketAddress(8080)); - PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); - ((PullCallback) mock.getArgument(4)).onSuccess(pullResult); - return pullResult; - } - }); - doReturn(new FindBrokerResult("127.0.0.1:10911", false)).when(mQClientFactory).findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean()); + Set messageQueueSet = new HashSet(); messageQueueSet.add(createPullRequest().getMessageQueue()); - pushConsumer.getDefaultMQPushConsumerImpl().updateTopicSubscribeInfo(topic, messageQueueSet); + pushConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + + pushConsumer.subscribe(topic, "*"); + pushConsumer.start(); } @After @@ -209,7 +192,8 @@ public class DefaultMQConsumerWithOpenTracingTest { assertThat(msg.getTopic()).isEqualTo(topic); assertThat(msg.getBody()).isEqualTo(new byte[]{'a'}); - assertThat(tracer.finishedSpans().size()).isEqualTo(1); + // wait until consumeMessageAfter hook of tracer is done surely. + waitAtMost(1, TimeUnit.SECONDS).until(() -> tracer.finishedSpans().size() == 1); MockSpan span = tracer.finishedSpans().get(0); assertThat(span.tags().get(Tags.MESSAGE_BUS_DESTINATION.getKey())).isEqualTo(topic); assertThat(span.tags().get(Tags.SPAN_KIND.getKey())).isEqualTo(Tags.SPAN_KIND_CONSUMER); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java new file mode 100644 index 0000000000..67ae194b88 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/trace/DefaultMQLitePullConsumerWithTraceTest.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.trace; + +import java.util.concurrent.ConcurrentMap; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.hook.SendMessageContext; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; +import org.apache.rocketmq.client.impl.consumer.RebalanceService; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.remoting.RPCHook; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultMQLitePullConsumerWithTraceTest { + + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(new ClientConfig()); + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private MQAdminImpl mQAdminImpl; + + private AsyncTraceDispatcher asyncTraceDispatcher; + private DefaultMQProducer traceProducer; + private RebalanceImpl rebalanceImpl; + private OffsetStore offsetStore; + private DefaultLitePullConsumerImpl litePullConsumerImpl; + private String consumerGroup = "LitePullConsumerGroup"; + private String topic = "LitePullConsumerTest"; + private String brokerName = "BrokerA"; + private String producerGroupTraceTemp = TopicValidator.RMQ_SYS_TRACE_TOPIC + System.currentTimeMillis(); + + private String customerTraceTopic = "rmq_trace_topic_12345"; + + @Before + public void init() throws Exception { + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); + field.setAccessible(true); + RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); + field = RebalanceService.class.getDeclaredField("waitInterval"); + field.setAccessible(true); + field.set(rebalanceService, 100); + } + + @Test + public void testSubscribe_PollMessageSuccess_WithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithDefaultTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_PollMessageSuccess_WithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumerWithCustomizedTraceTopic(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + + private DefaultLitePullConsumer createLitePullConsumerWithDefaultTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private DefaultLitePullConsumer createLitePullConsumerWithCustomizedTraceTopic() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setEnableMsgTrace(true); + litePullConsumer.setCustomizedTraceTopic(customerTraceTopic); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.subscribe(topic, "*"); + suppressUpdateTopicRouteInfoFromNameServer(litePullConsumer); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { + asyncTraceDispatcher = (AsyncTraceDispatcher) litePullConsumer.getTraceDispatcher(); + traceProducer = asyncTraceDispatcher.getTraceProducer(); + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(litePullConsumerImpl, mQClientFactory); + + PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper(); + field = PullAPIWrapper.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pullAPIWrapper, mQClientFactory); + + Field fieldTrace = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + fieldTrace.setAccessible(true); + fieldTrace.set(traceProducer.getDefaultMQProducerImpl(), mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + field = MQClientInstance.class.getDeclaredField("mQAdminImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQAdminImpl); + + field = DefaultLitePullConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl); + field = RebalanceImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(rebalanceImpl, mQClientFactory); + + offsetStore = spy(litePullConsumerImpl.getOffsetStore()); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("offsetStore"); + field.setAccessible(true); + field.set(litePullConsumerImpl, offsetStore); + + traceProducer.getDefaultMQProducerImpl().getmQClientFactory().registerProducer(producerGroupTraceTemp, traceProducer.getDefaultMQProducerImpl()); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + return pullResult; + } + }); + + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false)); + + doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); + + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSysFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + return topicRouteData; + } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("123"); + sendResult.setOffsetMsgId("123"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + private static void suppressUpdateTopicRouteInfoFromNameServer(DefaultLitePullConsumer litePullConsumer) throws IllegalAccessException { + DefaultLitePullConsumerImpl defaultLitePullConsumerImpl = (DefaultLitePullConsumerImpl) FieldUtils.readDeclaredField(litePullConsumer, "defaultLitePullConsumerImpl", true); + if (litePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + litePullConsumer.changeInstanceNameToPID(); + } + MQClientInstance mQClientFactory = spy(MQClientManager.getInstance().getOrCreateMQClientInstance(litePullConsumer, (RPCHook) FieldUtils.readDeclaredField(defaultLitePullConsumerImpl, "rpcHook", true))); + ConcurrentMap factoryTable = (ConcurrentMap) FieldUtils.readDeclaredField(MQClientManager.getInstance(), "factoryTable", true); + factoryTable.put(litePullConsumer.buildMQClientId(), mQClientFactory); + doReturn(false).when(mQClientFactory).updateTopicRouteInfoFromNameServer(anyString()); + } + +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java index bac12ea0ce..af5a705395 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceDataEncoderTest.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.client.trace; import org.apache.rocketmq.client.producer.LocalTransactionState; -import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.message.MessageType; import org.junit.Assert; import org.junit.Before; @@ -50,8 +49,7 @@ public class TraceDataEncoderTest { .append(245).append(TraceConstants.CONTENT_SPLITOR) .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) - .append(true).append(TraceConstants.CONTENT_SPLITOR) - .append(UtilAll.ipToIPv4Str(UtilAll.getIP())).append(TraceConstants.FIELD_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) .toString(); } @@ -104,7 +102,6 @@ public class TraceDataEncoderTest { traceBean.setTags("Tags"); traceBean.setMsgId("AC1415116D1418B4AAC217FE1B4E0000"); traceBean.setStoreHost("127.0.0.1:10911"); - traceBean.setClientHost("127.0.0.1@41700"); traceBean.setMsgType(MessageType.Trans_msg_Commit); traceBean.setTransactionId("transactionId"); traceBean.setTransactionState(LocalTransactionState.COMMIT_MESSAGE); diff --git a/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java index 51a1543ee5..b1fdbaf965 100644 --- a/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/trace/TraceViewTest.java @@ -17,7 +17,8 @@ package org.apache.rocketmq.client.trace; -import org.apache.rocketmq.common.UtilAll; +import org.apache.commons.codec.Charsets; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageType; import org.junit.Assert; import org.junit.Test; @@ -29,29 +30,30 @@ public class TraceViewTest { @Test public void testDecodeFromTraceTransData() { String messageBody = new StringBuilder() - .append("Pub").append(TraceConstants.CONTENT_SPLITOR) - .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR) - .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) - .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) - .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) - .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) - .append("Tags").append(TraceConstants.CONTENT_SPLITOR) - .append("Keys").append(TraceConstants.CONTENT_SPLITOR) - .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) - .append(26).append(TraceConstants.CONTENT_SPLITOR) - .append(245).append(TraceConstants.CONTENT_SPLITOR) - .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) - .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) - .append(true).append(TraceConstants.CONTENT_SPLITOR) - .append(UtilAll.ipToIPv4Str(UtilAll.getIP())).append(TraceConstants.FIELD_SPLITOR) - .toString(); + .append("Pub").append(TraceConstants.CONTENT_SPLITOR) + .append(System.currentTimeMillis()).append(TraceConstants.CONTENT_SPLITOR) + .append("DefaultRegion").append(TraceConstants.CONTENT_SPLITOR) + .append("PID-test").append(TraceConstants.CONTENT_SPLITOR) + .append("topic-test").append(TraceConstants.CONTENT_SPLITOR) + .append("AC1415116D1418B4AAC217FE1B4E0000").append(TraceConstants.CONTENT_SPLITOR) + .append("Tags").append(TraceConstants.CONTENT_SPLITOR) + .append("Keys").append(TraceConstants.CONTENT_SPLITOR) + .append("127.0.0.1:10911").append(TraceConstants.CONTENT_SPLITOR) + .append(26).append(TraceConstants.CONTENT_SPLITOR) + .append(245).append(TraceConstants.CONTENT_SPLITOR) + .append(MessageType.Normal_Msg.ordinal()).append(TraceConstants.CONTENT_SPLITOR) + .append("0A9A002600002A9F0000000000002329").append(TraceConstants.CONTENT_SPLITOR) + .append(true).append(TraceConstants.FIELD_SPLITOR) + .toString(); + MessageExt message = new MessageExt(); + message.setBody(messageBody.getBytes(Charsets.UTF_8)); String key = "AC1415116D1418B4AAC217FE1B4E0000"; - List traceViews = TraceView.decodeFromTraceTransData(key, messageBody); + List traceViews = TraceView.decodeFromTraceTransData(key, message); Assert.assertEquals(traceViews.size(), 1); Assert.assertEquals(traceViews.get(0).getMsgId(), key); key = "AD4233434334AAC217FEFFD0000"; - traceViews = TraceView.decodeFromTraceTransData(key, messageBody); + traceViews = TraceView.decodeFromTraceTransData(key, message); Assert.assertEquals(traceViews.size(), 0); } } diff --git a/common/pom.xml b/common/pom.xml index 068de26dcb..ac1d086b4c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 0d4aff0b0f..48cd1582cc 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.common; public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_8_0.ordinal(); + public static final int CURRENT_VERSION = Version.V4_9_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java index 200dec2481..1fe9aa106e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/PermName.java @@ -20,10 +20,10 @@ public class PermName { public static final int PERM_PRIORITY = 0x1 << 3; public static final int PERM_READ = 0x1 << 2; public static final int PERM_WRITE = 0x1 << 1; - public static final int PERM_INHERIT = 0x1 << 0; + public static final int PERM_INHERIT = 0x1; public static String perm2String(final int perm) { - final StringBuffer sb = new StringBuffer("---"); + final StringBuilder sb = new StringBuilder("---"); if (isReadable(perm)) { sb.replace(0, 1, "R"); } diff --git a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java index e72fb82601..f57df26d6f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java +++ b/common/src/main/java/org/apache/rocketmq/common/hook/FilterCheckHook.java @@ -20,7 +20,7 @@ package org.apache.rocketmq.common.hook; import java.nio.ByteBuffer; public interface FilterCheckHook { - public String hookName(); + String hookName(); - public boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); + boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); } diff --git a/distribution/bin/dledger/fast-try.sh b/distribution/bin/dledger/fast-try.sh index ff8a96036d..acdde71b8d 100644 --- a/distribution/bin/dledger/fast-try.sh +++ b/distribution/bin/dledger/fast-try.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -20,25 +20,25 @@ CURRENT_DIR="$(cd "$(dirname "$0")"; pwd)" RMQ_DIR=$CURRENT_DIR/../.. cd $RMQ_DIR -function startNameserver() { +startNameserver() { export JAVA_OPT_EXT=" -Xms512m -Xmx512m " nohup bin/mqnamesrv & } -function startBroker() { +startBroker() { export JAVA_OPT_EXT=" -Xms1g -Xmx1g " conf_name=$1 nohup bin/mqbroker -c $conf_name & } -function stopNameserver() { +stopNameserver() { PIDS=$(ps -ef|grep java|grep NamesrvStartup|grep -v grep|awk '{print $2}') if [ ! -z "$PIDS" ]; then kill -s TERM $PIDS fi } -function stopBroker() { +stopBroker() { conf_name=$1 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') i=1 @@ -46,7 +46,7 @@ function stopBroker() { do echo "Waiting to kill ..." kill -s TERM $PIDS - ((i=$i+1)) + i=`expr $i + 1` sleep 2 PIDS=$(ps -ef|grep java|grep BrokerStartup|grep $conf_name|grep -v grep|awk '{print $2}') done @@ -56,7 +56,7 @@ function stopBroker() { fi } -function stopAll() { +stopAll() { ps -ef|grep java|grep BrokerStartup|grep -v grep|awk '{print $2}'|xargs kill stopNameserver stopBroker ./conf/dledger/broker-n0.conf @@ -64,18 +64,18 @@ function stopAll() { stopBroker ./conf/dledger/broker-n2.conf } -function startAll() { +startAll() { startNameserver startBroker ./conf/dledger/broker-n0.conf startBroker ./conf/dledger/broker-n1.conf startBroker ./conf/dledger/broker-n2.conf } -function checkConf() { +checkConf() { if [ ! -f ./conf/dledger/broker-n0.conf -o ! -f ./conf/dledger/broker-n1.conf -o ! -f ./conf/dledger/broker-n2.conf ]; then echo "Make sure the ./conf/dledger/broker-n0.conf, ./conf/dledger/broker-n1.conf, ./conf/dledger/broker-n2.conf exists" - exit -1 - fi + exit 1 + fi } @@ -83,7 +83,7 @@ function checkConf() { ## Main if [ $# -lt 1 ]; then echo "Usage: sh $0 start|stop" - exit -1 + exit 1 fi action=$1 checkConf diff --git a/distribution/bin/runbroker.cmd b/distribution/bin/runbroker.cmd index eab7e30f9d..753a11e9a4 100644 --- a/distribution/bin/runbroker.cmd +++ b/distribution/bin/runbroker.cmd @@ -36,7 +36,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:+AlwaysPreTouch" set "JAVA_OPT=%JAVA_OPT% -XX:MaxDirectMemorySize=15g" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages -XX:-UseBiasedLocking" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp %CLASSPATH%" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/bin/runserver.cmd b/distribution/bin/runserver.cmd index 48e32bf2d7..76865f7a16 100644 --- a/distribution/bin/runserver.cmd +++ b/distribution/bin/runserver.cmd @@ -31,7 +31,7 @@ set "JAVA_OPT=%JAVA_OPT% -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollect set "JAVA_OPT=%JAVA_OPT% -verbose:gc -Xloggc:"%USERPROFILE%\rmq_srv_gc.log" -XX:+PrintGCDetails" set "JAVA_OPT=%JAVA_OPT% -XX:-OmitStackTraceInFastThrow" set "JAVA_OPT=%JAVA_OPT% -XX:-UseLargePages" -set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib" +set "JAVA_OPT=%JAVA_OPT% -Djava.ext.dirs=%BASE_DIR%lib;%JAVA_HOME%\jre\lib\ext" set "JAVA_OPT=%JAVA_OPT% -cp "%CLASSPATH%"" "%JAVA%" %JAVA_OPT% %* \ No newline at end of file diff --git a/distribution/pom.xml b/distribution/pom.xml index d5ab667229..a5698bca1c 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index fd36fd1c31..d924ce1d7d 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -446,7 +446,7 @@ public class ScheduledMessageConsumer { public ConsumeConcurrentlyStatus consumeMessage(List messages, ConsumeConcurrentlyContext context) { for (MessageExt message : messages) { // Print approximate delay time period - System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getStoreTimestamp()) + "ms later"); + System.out.println("Receive message[msgId=" + message.getMsgId() + "] " + (System.currentTimeMillis() - message.getBornTimestamp()) + "ms later"); } return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; } diff --git a/docs/cn/design.md b/docs/cn/design.md index dcd39026f2..1c32521ed5 100644 --- a/docs/cn/design.md +++ b/docs/cn/design.md @@ -58,7 +58,7 @@ code |int | 请求操作码，应答方根据不同的请求码进行不同的 language | LanguageCode | 请求方实现的语言 | 应答方实现的语言 version | int | 请求方程序的版本 | 应答方程序的版本 opaque | int |相当于requestId，在同一个连接上的不同请求标识码，与响应消息中的相对应 | 应答不做修改直接返回 -flag | int | 区分是普通RPC还是onewayRPC得标志 | 区分是普通RPC还是onewayRPC得标志 +flag | int | 区分是普通RPC还是onewayRPC的标志 | 区分是普通RPC还是onewayRPC的标志 remark | String | 传输自定义文本信息 | 传输自定义文本信息 extFields | HashMap | 请求自定义扩展信息 | 响应自定义扩展信息 diff --git a/docs/cn/operation.md b/docs/cn/operation.md index d8c0bb151d..bc2d38527f 100644 --- a/docs/cn/operation.md +++ b/docs/cn/operation.md @@ -26,7 +26,7 @@ The Name Server boot success... $ nohup sh bin/mqbroker -n localhost:9876 & ### 验证Name Server 是否启动成功，例如Broker的IP为：192.168.1.2，且名称为broker-a -$ tail -f ~/logs/rocketmqlogs/Broker.log +$ tail -f ~/logs/rocketmqlogs/broker.log The broker[broker-a, 192.169.1.2:10911] boot success... ``` diff --git a/docs/en/Design_LoadBlancing.md b/docs/en/Design_LoadBlancing.md index e6bb66f8ac..971e8b5dd4 100644 --- a/docs/en/Design_LoadBlancing.md +++ b/docs/en/Design_LoadBlancing.md @@ -1,22 +1,24 @@ ## Load Balancing Load balancing in RocketMQ is accomplished on Client side. Specifically, it can be divided into load balancing at Producer side when sending messages and load balancing at Constumer side when subscribing messages. -### 1 Producer Load Balancing +### Producer Load Balancing When the Producer sends a message, it will first find the specified TopicPublishInfo according to Topic. After getting the routing information of TopicPublishInfo, the RocketMQ client will select a queue (MessageQueue) from the messageQueue List in TopicPublishInfo to send the message by default.Specific fault-tolerant strategies are defined in the MQFaultStrategy class. Here is a sendLatencyFaultEnable switch variable, which, if turned on, filters out the Broker agent of not available on the basis of randomly gradually increasing modular arithmetic selection. The so-called "latencyFault Tolerance" refers to a certain period of time to avoid previous failures. For example, if the latency of the last request exceeds 550 Lms, it will evade 3000 Lms; if it exceeds 1000L, it will evade 60000 L; if it is closed, it will choose a queue (MessageQueue) to send messages by randomly gradually increasing modular arithmetic, and the latencyFault Tolerance mechanism is the key to achieve high availability of message sending. -### 2 Consumer Load Balancing +### Consumer Load Balancing In RocketMQ, the two consumption modes (Push/Pull) on the Consumer side are both based on the pull mode to get the message, while in the Push mode it is only a kind of encapsulation of the pull mode, which is essentially implemented as the message pulling thread after pulling a batch of messages from the server. After submitting to the message consuming thread pool, it continues to try again to pull the message to the server. If the message is not pulled, the pull is delayed and continues. In both pull mode based consumption patterns (Push/Pull), the Consumer needs to know which message queue - queue from the Broker side to get the message. Therefore, it is necessary to do load balancing on the Consumer side, that is, which Consumer consumption is allocated to the same ConsumerGroup by more than one MessageQueue on the Broker side. - 1， Heartbeat Packet Sending on Consumer side +#### 1 Heartbeat Packet Sending on Consumer side After Consumer is started, it continuously sends heartbeat packets to all Broker instances in the RocketMQ cluster via timing task (which contains the message consumption group name, subscription relationship collection,Message communication mode and the value of the client id,etc). After receiving the heartbeat message from Consumer, Broker side maintains it in Consumer Manager's local caching variable—consumerTable, At the same time, the encapsulated client network channel information is stored in the local caching variable—channelInfoTable, which can provide metadata information for the later load balancing of Consumer. -2，Core Class for Load Balancing on Consumer side—RebalanceImpl +#### 2 Core Class for Load Balancing on Consumer side—RebalanceImpl Starting the MQClientInstance instance in the startup process of the Consumer instance will complete the start of the load balancing service thread-RebalanceService (executed every 20 s). By looking at the source code, we can find that the run () method of the RebalanceService thread calls the rebalanceByTopic () method of the RebalanceImpl class, which is the core of the Consumer end load balancing. Here, rebalanceByTopic () method will do different logical processing depending on whether the consumer communication type is "broadcast mode" or "cluster mode". Here we mainly look at the main processing flow in cluster mode: -(1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. -(2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); -(3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). +##### 1) Get the message consumption queue set (mqSet) under the Topic from the local cache variable—topicSubscribeInfoTable of the rebalanceImpl instance. +##### 2) Call mQClientFactory. findConsumerIdList () method to send a RPC communication request to Broker side to obtain the consumer Id list under the consumer group based on the parameters of topic and consumer group (consumer table constructed by Broker side based on the heartbeat data reported by the front consumer side responds and returns, business request code: GET_CONSUMER_LIST_BY_GROUP); +##### 3) First, the message consumption queue and the consumer Id under Topic are sorted, then the message queue to be pulled is calculated by using the message queue allocation strategy algorithm (default: the average allocation algorithm of the message queue). The average allocation algorithm here is similar to the paging algorithm. It ranks all MessageQueues like records. It ranks all consumers like pages. It calculates the average size of each page and the range of each page record. Finally, it traverses the whole range and calculates the records that the current consumer should allocate to (MessageQueue here). ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_8.png) -(4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. + + +##### 4) Then, the updateProcessQueueTableInRebalance () method is invoked, which first compares the allocated message queue set (mqSet) with processQueueTable for filtering. ![Image text](https://github.com/apache/rocketmq/raw/develop/docs/cn/image/rocketmq_design_9.png) - The red part of the processQueueTable annotation in the figure above @@ -37,6 +39,6 @@ Starting the MQClientInstance instance in the startup process of the Consumer in removeUnnecessaryMessageQueue () method to try to remove Entry as above; -Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. Among them, we can focus on the contrast, RebalancePushImpl and RebalancePullImpl two implementation classes dispatchPullRequest () method is different, the method in RebalancePullImpl class is empty, thus answering the last question in the previous article. +Finally, a ProcessQueue object is created for each MessageQueue in the filtered message queue set (mqSet) and stored in the processQueueTable queue of RebalanceImpl (where the computePullFromWhere (MessageQueue mq) method of the RebalanceImpl instance is invoked to obtain the next progress consumption value offset of the MessageQueue object, which is then populated into the attribute of pullRequest object to be created next time.), and create pull request object—pullRequest to add to pull list—pullRequestList, and finally execute dispatchPullRequest () method. PullRequest object of Pull message is put into the blocking queue pullRequestQueue of PullMessageService service thread in turn, and the request of Pull message is sent to Broker end after the service thread takes out. The core design idea of message consumption queue is that a message consumption queue can only be consumed by one consumer in the same consumer group at the same time, and a message consumer can consume multiple message queues at the same time. diff --git a/example/pom.xml b/example/pom.xml index ca5189378b..b392d9a10c 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java index b6c6ae410f..8a6429b2aa 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java @@ -54,7 +54,7 @@ public class Consumer { final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 20; final String groupPrefix = commandLine.hasOption('g') ? commandLine.getOptionValue('g').trim() : "benchmark_consumer"; - final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "true"; + final String isSuffixEnable = commandLine.hasOption('p') ? commandLine.getOptionValue('p').trim() : "false"; final String filterType = commandLine.hasOption('f') ? commandLine.getOptionValue('f').trim() : null; final String expression = commandLine.hasOption('e') ? commandLine.getOptionValue('e').trim() : null; final double failRate = commandLine.hasOption('r') ? Double.parseDouble(commandLine.getOptionValue('r').trim()) : 0.0; @@ -190,7 +190,7 @@ public class Consumer { opt.setRequired(false); options.addOption(opt); - opt = new Option("p", "group prefix enable", true, "Consumer group name, Default: false"); + opt = new Option("p", "group suffix enable", true, "Consumer group suffix enable, Default: false"); opt.setRequired(false); options.addOption(opt); diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java index d1fa5743ae..6975ab58b0 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Producer.java @@ -17,12 +17,14 @@ package org.apache.rocketmq.example.benchmark; import java.io.UnsupportedEncodingException; +import java.util.Arrays; import java.util.LinkedList; import java.util.Random; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.cli.CommandLine; @@ -58,9 +60,10 @@ public class Producer { final int tagCount = commandLine.hasOption('l') ? Integer.parseInt(commandLine.getOptionValue('l')) : 0; final boolean msgTraceEnable = commandLine.hasOption('m') && Boolean.parseBoolean(commandLine.getOptionValue('m')); final boolean aclEnable = commandLine.hasOption('a') && Boolean.parseBoolean(commandLine.getOptionValue('a')); + final long messageNum = commandLine.hasOption('q') ? Long.parseLong(commandLine.getOptionValue('q')) : 0; - System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s%n", - topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable); + System.out.printf("topic: %s threadCount: %d messageSize: %d keyEnable: %s propertySize: %d tagCount: %d traceEnable: %s aclEnable: %s messageQuantity: %d%n", + topic, threadCount, messageSize, keyEnable, propertySize, tagCount, msgTraceEnable, aclEnable, messageNum); final InternalLogger log = ClientLogger.getLog(); @@ -72,6 +75,16 @@ public class Producer { final LinkedList snapshotList = new LinkedList(); + final long[] msgNums = new long[threadCount]; + + if (messageNum > 0) { + Arrays.fill(msgNums, messageNum / threadCount); + long mod = messageNum % threadCount; + if (mod > 0) { + msgNums[0] += mod; + } + } + timer.scheduleAtFixedRate(new TimerTask() { @Override public void run() { @@ -85,14 +98,7 @@ public class Producer { timer.scheduleAtFixedRate(new TimerTask() { private void printStats() { if (snapshotList.size() >= 10) { - Long[] begin = snapshotList.getFirst(); - Long[] end = snapshotList.getLast(); - - final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); - final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); - - System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", - System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + doPrintStats(snapshotList, statsBenchmark, false); } } @@ -120,9 +126,14 @@ public class Producer { producer.start(); for (int i = 0; i < threadCount; i++) { + final long msgNumLimit = msgNums[i]; + if (messageNum > 0 && msgNumLimit == 0) { + break; + } sendThreadPool.execute(new Runnable() { @Override public void run() { + int num = 0; while (true) { try { final Message msg; @@ -198,10 +209,28 @@ public class Producer { } catch (InterruptedException ignored) { } } + if (messageNum > 0 && ++num >= msgNumLimit) { + break; + } } } }); } + try { + sendThreadPool.shutdown(); + sendThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + timer.cancel(); + if (snapshotList.size() > 1) { + doPrintStats(snapshotList, statsBenchmark, true); + } else { + System.out.printf("[Complete] Send Total: %d Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + statsBenchmark.getSendRequestFailedCount().get(), statsBenchmark.getReceiveResponseFailedCount().get()); + } + producer.shutdown(); + } catch (InterruptedException e) { + log.error("[Exit] Thread Interrupted Exception", e); + } } public static Options buildCommandlineOptions(final Options options) { @@ -233,6 +262,10 @@ public class Producer { opt.setRequired(false); options.addOption(opt); + opt = new Option("q", "messageQuantity", true, "Send message quantity, Default: 0, running forever"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -249,6 +282,23 @@ public class Producer { return msg; } + + private static void doPrintStats(final LinkedList snapshotList, final StatsBenchmarkProducer statsBenchmark, boolean done) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = (end[5] - begin[5]) / (double) (end[3] - begin[3]); + + if (done) { + System.out.printf("[Complete] Send Total: %d Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + statsBenchmark.getSendRequestSuccessCount().get() + statsBenchmark.getSendRequestFailedCount().get(), + sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } else { + System.out.printf("Current Time: %s Send TPS: %d Max RT(ms): %d Average RT(ms): %7.3f Send Failed: %d Response Failed: %d%n", + System.currentTimeMillis(), sendTps, statsBenchmark.getSendMessageMaxRT().get(), averageRT, end[2], end[4]); + } + } } class StatsBenchmarkProducer { diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java index 4b737f7010..d7a2c70afb 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/PopPushConsumer.java @@ -55,6 +55,7 @@ public class PopPushConsumer { System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; }); + consumer.setClientRebalance(false); consumer.start(); System.out.printf("Consumer Started.%n"); } diff --git a/filter/pom.xml b/filter/pom.xml index c12dd1fb28..41bda9d0d2 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java index 53f7e1c297..c10250e3bb 100644 --- a/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java +++ b/filter/src/main/java/org/apache/rocketmq/filter/parser/SimpleCharStream.java @@ -36,8 +36,8 @@ public class SimpleCharStream { * Position in buffer. */ public int bufpos = -1; - protected int bufline[]; - protected int bufcolumn[]; + protected int[] bufline; + protected int[] bufcolumn; protected int column = 0; protected int line = 1; @@ -62,8 +62,8 @@ public class SimpleCharStream { protected void ExpandBuff(boolean wrapAround) { char[] newbuffer = new char[bufsize + 2048]; - int newbufline[] = new int[bufsize + 2048]; - int newbufcolumn[] = new int[bufsize + 2048]; + int[] newbufline = new int[bufsize + 2048]; + int[] newbufcolumn = new int[bufsize + 2048]; try { if (wrapAround) { diff --git a/logappender/pom.xml b/logappender/pom.xml index a6e979a83e..cac520b610 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index 77d0f29a97..8ebb243045 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java index 487682cd44..0dc81d74e6 100755 --- a/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java +++ b/logging/src/main/java/org/apache/rocketmq/logging/inner/Level.java @@ -116,7 +116,6 @@ public class Level implements Serializable { if (s.equals(OFF_NAME)) { return Level.OFF; } - if (s.equals(INFO_NAME)) { return Level.INFO; } diff --git a/namesrv/pom.xml b/namesrv/pom.xml index c5d6d05bc2..01ef5b60fe 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java index 44a2b6f261..f8bc55e7aa 100644 --- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -140,6 +140,11 @@ public class DefaultRequestProcessor extends AsyncNettyRequestProcessor implemen final PutKVConfigRequestHeader requestHeader = (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class); + if (requestHeader.getNamespace() == null || requestHeader.getKey() == null) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("namespace or key is null"); + return response; + } this.namesrvController.getKvConfigManager().putKVConfig( requestHeader.getNamespace(), requestHeader.getKey(), diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 7f0e937996..4688c04043 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 7c2a8f4ce9..c6e039e583 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -439,6 +439,12 @@ 3.10.0 test + + org.awaitility + awaitility + 4.1.0 + test + @@ -536,7 +542,7 @@ com.alibaba fastjson - 1.2.69 + 1.2.76 org.javassist diff --git a/remoting/pom.xml b/remoting/pom.xml index d9dab9efa6..765e1a93bc 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java index a16940e561..d936c3bf69 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/common/RemotingUtil.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.remoting.common; import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelFutureListener; +import java.io.File; import java.io.IOException; import java.lang.reflect.Method; import java.net.Inet6Address; @@ -98,6 +99,10 @@ public class RemotingUtil { ArrayList ipv6Result = new ArrayList(); while (enumeration.hasMoreElements()) { final NetworkInterface networkInterface = enumeration.nextElement(); + if (isBridge(networkInterface)) { + continue; + } + final Enumeration en = networkInterface.getInetAddresses(); while (en.hasMoreElements()) { final InetAddress address = en.nextElement(); @@ -160,6 +165,19 @@ public class RemotingUtil { return sb.toString(); } + private static boolean isBridge(NetworkInterface networkInterface) { + try { + if (isLinuxPlatform()) { + String interfaceName = networkInterface.getName(); + File file = new File("/sys/class/net/" + interfaceName + "/bridge"); + return file.exists(); + } + } catch (SecurityException e) { + //Ignore + } + return false; + } + public static SocketChannel connect(SocketAddress remote) { return connect(remote, 1000 * 5); } diff --git a/srvutil/pom.xml b/srvutil/pom.xml index b95808b7c2..9c4544c0b2 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index ed54db1b24..23e9763d81 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index c13ad4cfa2..dd520f4859 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -152,6 +152,8 @@ public class MessageStoreConfig { private boolean isEnableBatchPush = false; + private boolean enableScheduleMessageStats = true; + public boolean isDebugLockEnable() { return debugLockEnable; } @@ -722,4 +724,12 @@ public class MessageStoreConfig { public void setEnableBatchPush(boolean enableBatchPush) { isEnableBatchPush = enableBatchPush; } + + public boolean isEnableScheduleMessageStats() { + return enableScheduleMessageStats; + } + + public void setEnableScheduleMessageStats(boolean enableScheduleMessageStats) { + this.enableScheduleMessageStats = enableScheduleMessageStats; + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java index 9a6e7a78a1..9241ffe423 100644 --- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java @@ -426,17 +426,18 @@ public class DLedgerCommitLog extends CommitLog { AppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(msg); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config long elapsedTimeInLock; long queueOffset; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - encodeResult = this.messageSerializer.serialize(msg); queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); - encodeResult.setQueueOffsetKey(queueOffset); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult.status)); - } + encodeResult.setQueueOffsetKey(queueOffset, false); AppendEntryRequest request = new AppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -542,6 +543,12 @@ public class DLedgerCommitLog extends CommitLog { BatchAppendFuture dledgerFuture; EncodeResult encodeResult; + encodeResult = this.messageSerializer.serialize(messageExtBatch); + if (encodeResult.status != AppendMessageStatus.PUT_OK) { + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult + .status)); + } + putMessageLock.lock(); //spin or ReentrantLock ,depending on store config msgIdBuilder.setLength(0); long elapsedTimeInLock; @@ -549,12 +556,8 @@ public class DLedgerCommitLog extends CommitLog { long msgNum = 0; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - encodeResult = this.messageSerializer.serialize(messageExtBatch); - queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); - if (encodeResult.status != AppendMessageStatus.PUT_OK) { - return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, new AppendMessageResult(encodeResult - .status)); - } + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); BatchAppendEntryRequest request = new BatchAppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -664,7 +667,7 @@ public class DLedgerCommitLog extends CommitLog { try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); - encodeResult.setQueueOffsetKey(queueOffset); + encodeResult.setQueueOffsetKey(queueOffset, false); AppendEntryRequest request = new AppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -779,7 +782,8 @@ public class DLedgerCommitLog extends CommitLog { long msgNum = 0; try { beginTimeInDledgerLock = this.defaultMessageStore.getSystemClock().now(); - queueOffset = topicQueueTable.get(encodeResult.queueOffsetKey); + queueOffset = getQueueOffsetByKey(encodeResult.queueOffsetKey, tranType); + encodeResult.setQueueOffsetKey(queueOffset, true); BatchAppendEntryRequest request = new BatchAppendEntryRequest(); request.setGroup(dLedgerConfig.getGroup()); request.setRemoteId(dLedgerServer.getMemberState().getSelfId()); @@ -957,8 +961,15 @@ public class DLedgerCommitLog extends CommitLog { this.queueOffsetKey = queueOffsetKey; } - public void setQueueOffsetKey(long offset) { - data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset); + public void setQueueOffsetKey(long offset, boolean isBatch) { + if (!isBatch) { + this.data.putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset); + return; + } + + for (byte[] data : batchData) { + ByteBuffer.wrap(data).putLong(MessageDecoder.QUEUE_OFFSET_POSITION, offset++); + } } public byte[] getData() { @@ -977,8 +988,6 @@ public class DLedgerCommitLog extends CommitLog { // The maximum length of the message private final int maxMessageSize; - // Build Message Key - private final StringBuilder keyBuilder = new StringBuilder(); MessageSerializer(final int size) { this.maxMessageSize = size; @@ -1079,17 +1088,7 @@ public class DLedgerCommitLog extends CommitLog { } public EncodeResult serialize(final MessageExtBatch messageExtBatch) { - keyBuilder.setLength(0); - keyBuilder.append(messageExtBatch.getTopic()); - keyBuilder.append('-'); - keyBuilder.append(messageExtBatch.getQueueId()); - String key = keyBuilder.toString(); - - Long queueOffset = DLedgerCommitLog.this.topicQueueTable.get(key); - if (null == queueOffset) { - queueOffset = 0L; - DLedgerCommitLog.this.topicQueueTable.put(key, queueOffset); - } + String key = messageExtBatch.getTopic() + "-" + messageExtBatch.getQueueId(); int totalMsgLen = 0; ByteBuffer messagesByteBuff = messageExtBatch.wrap(); @@ -1154,7 +1153,7 @@ public class DLedgerCommitLog extends CommitLog { // 5 FLAG msgStoreItemMemory.putInt(flag); // 6 QUEUEOFFSET - msgStoreItemMemory.putLong(queueOffset++); + msgStoreItemMemory.putLong(0L); // 7 PHYSICALOFFSET msgStoreItemMemory.putLong(0); // 8 SYSFLAG @@ -1210,6 +1209,7 @@ public class DLedgerCommitLog extends CommitLog { this.sbr = sbr; } + @Override public synchronized void release() { super.release(); if (sbr != null) { diff --git a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java index ee994c37a6..9057ebefdf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java +++ b/store/src/main/java/org/apache/rocketmq/store/schedule/ScheduleMessageService.java @@ -115,6 +115,7 @@ public class ScheduleMessageService extends ConfigManager { public void start() { if (started.compareAndSet(false, true)) { + super.load(); this.timer = new Timer("ScheduleMessageTimerThread", true); for (Map.Entry entry : this.delayLevelTable.entrySet()) { Integer level = entry.getKey(); @@ -330,6 +331,12 @@ public class ScheduleMessageService extends ConfigManager { if (putMessageResult != null && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + if (ScheduleMessageService.this.defaultMessageStore.getMessageStoreConfig().isEnableScheduleMessageStats()) { + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + ScheduleMessageService.this.defaultMessageStore.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); + } continue; } else { // XXX: warn and notify me diff --git a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java index fd860e6b9d..fa3c6bfcd8 100644 --- a/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/schedule/ScheduleMessageServiceTest.java @@ -40,6 +40,9 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.BROKER_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_NUMS; +import static org.apache.rocketmq.store.stats.BrokerStatsManager.TOPIC_PUT_SIZE; import static org.assertj.core.api.Assertions.assertThat; @@ -112,6 +115,10 @@ public class ScheduleMessageServiceTest { @Test public void deliverDelayedMessageTimerTaskTest() throws Exception { + assertThat(messageStore.getMessageStoreConfig().isEnableScheduleMessageStats()).isTrue(); + + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic)).isNull(); + MessageExtBrokerInner msg = buildMessage(); int realQueueId = msg.getQueueId(); // set delayLevel,and send delay message @@ -141,6 +148,10 @@ public class ScheduleMessageServiceTest { // now,found the message assertThat(messageResult.getStatus()).isEqualTo(GetMessageStatus.FOUND); + // get the stats change + assertThat(messageStore.getBrokerStatsManager().getStatsItem(BROKER_PUT_NUMS, brokerConfig.getBrokerClusterName()).getValue().get()).isEqualTo(1); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_NUMS, topic).getValue().get()).isEqualTo(1L); + assertThat(messageStore.getBrokerStatsManager().getStatsItem(TOPIC_PUT_SIZE, topic).getValue().get()).isEqualTo(messageResult.getBufferTotalSize()); // get the message body ByteBuffer byteBuffer = ByteBuffer.allocate(messageResult.getBufferTotalSize()); diff --git a/test/pom.xml b/test/pom.xml index 851d9ab9aa..41296501c2 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java new file mode 100644 index 0000000000..036f60e1db --- /dev/null +++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopConsumer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.client.rmq; + +import org.apache.rocketmq.test.listener.AbstractListener; + +public class RMQPopConsumer extends RMQNormalConsumer { + public RMQPopConsumer(String nsAddr, String topic, String subExpression, + String consumerGroup, AbstractListener listner) { + super(nsAddr, topic, subExpression, consumerGroup, listner); + } + + @Override + public void create() { + super.create(); + consumer.setClientRebalance(false); + } +} diff --git a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java index 4850846266..d530db98b0 100644 --- a/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java +++ b/test/src/main/java/org/apache/rocketmq/test/factory/ConsumerFactory.java @@ -22,6 +22,7 @@ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.test.client.rmq.RMQBroadCastConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; import org.apache.rocketmq.test.client.rmq.RMQSqlConsumer; import org.apache.rocketmq.test.listener.AbstractListener; @@ -62,6 +63,15 @@ public class ConsumerFactory { consumer.start(); return consumer; } + public static RMQPopConsumer getRMQPopConsumer(String nsAddr, String consumerGroup, + String topic, String subExpression, + AbstractListener listener) { + RMQPopConsumer consumer = new RMQPopConsumer(nsAddr, topic, subExpression, + consumerGroup, listener); + consumer.create(); + consumer.start(); + return consumer; + } public static DefaultMQPullConsumer getRMQPullConsumer(String nsAddr, String consumerGroup) throws Exception { DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java index 44db782b5c..e08967eb91 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/FileUtil.java @@ -33,7 +33,7 @@ public class FileUtil { this.fileName = fileName; } - public static void main(String args[]) { + public static void main(String[] args) { String filePath = FileUtil.class.getResource("/").getPath(); String fileName = "test.txt"; FileUtil fileUtil = new FileUtil(filePath, fileName); diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java index 6edeecadb2..0c24427d92 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQWait.java @@ -76,7 +76,7 @@ public class MQWait { } } - public static void main(String args[]) { + public static void main(String[] args) { long start = System.currentTimeMillis(); MQWait.setCondition(new Condition() { diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java index 1c2bdac305..41cacb684c 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtil.java @@ -100,14 +100,14 @@ public final class RandomUtil { return n + res % (m - n); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; return (char) (getIntegerBetween(arg[c], arg[c + 1])); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -116,17 +116,17 @@ public final class RandomUtil { } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithNumAndCha(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1, '0', '9' + 1}; return getString(n, arg); } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java index 9eca28bbe7..3f71176d6f 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/RandomUtils.java @@ -45,16 +45,16 @@ public class RandomUtils { } public static String getStringWithNumber(int n) { - int arg[] = new int[] {'0', '9' + 1}; + int[] arg = new int[] {'0', '9' + 1}; return getString(n, arg); } public static String getStringWithCharacter(int n) { - int arg[] = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; + int[] arg = new int[] {'a', 'z' + 1, 'A', 'Z' + 1}; return getString(n, arg); } - private static String getString(int n, int arg[]) { + private static String getString(int n, int[] arg) { StringBuilder res = new StringBuilder(); for (int i = 0; i < n; i++) { res.append(getChar(arg)); @@ -62,7 +62,7 @@ public class RandomUtils { return res.toString(); } - private static char getChar(int arg[]) { + private static char getChar(int[] arg) { int size = arg.length; int c = rd.nextInt(size / 2); c = c * 2; diff --git a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java index 965d2ee670..69bfd8f7d1 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/VerifyUtils.java @@ -140,7 +140,7 @@ public class VerifyUtils { return rtExpect; } - public static void main(String args[]) { + public static void main(String[] args) { verifyBalance(400, 0.1f, 230, 190); } } diff --git a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java index 82ab461aa5..bdd991a335 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/data/collect/impl/ListDataCollectorImpl.java @@ -43,7 +43,7 @@ public class ListDataCollectorImpl implements DataCollector { return datas; } - public void resetData() { + public synchronized void resetData() { datas.clear(); unlockIncrement(); } @@ -67,7 +67,7 @@ public class ListDataCollectorImpl implements DataCollector { return Collections.frequency(datas, data) == 1; } - public Collection getAllDataWithoutDuplicate() { + public synchronized Collection getAllDataWithoutDuplicate() { return new HashSet(datas); } @@ -81,7 +81,7 @@ public class ListDataCollectorImpl implements DataCollector { return res; } - public void removeData(Object data) { + public synchronized void removeData(Object data) { datas.remove(data); } diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index a32da0320f..82420105e0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -133,6 +133,7 @@ public class IntegrationTestBase { brokerConfig.setBrokerIP1("127.0.0.1"); brokerConfig.setNamesrvAddr(nsAddr); brokerConfig.setEnablePropertyFilter(true); + brokerConfig.setLoadBalancePollNameServerInterval(500); storeConfig.setStorePathRootDir(baseDir); storeConfig.setStorePathCommitLog(baseDir + SEP + "commitlog"); storeConfig.setMappedFileSizeCommitLog(COMMIT_LOG_SIZE); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java index 4c31f6a6ce..7b2f09ed02 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/balance/NormalMsgStaticBalanceIT.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.test.client.consumer.balance; -import org.apache.log4j.Logger; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -29,11 +28,13 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import static com.google.common.truth.Truth.assertThat; public class NormalMsgStaticBalanceIT extends BaseConf { - private static Logger logger = Logger.getLogger(NormalMsgStaticBalanceIT.class); + private static Logger logger = LoggerFactory.getLogger(NormalMsgStaticBalanceIT.class); private RMQNormalProducer producer = null; private String topic = null; @@ -75,13 +76,12 @@ public class NormalMsgStaticBalanceIT extends BaseConf { @Test public void testFourConsumersBalance() { int msgSize = 600; - RMQNormalConsumer consumer1 = getConsumer(nsAddr, topic, "*", new RMQNormalListener()); - RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); - RMQNormalConsumer consumer3 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); - RMQNormalConsumer consumer4 = getConsumer(nsAddr, consumer1.getConsumerGroup(), topic, - "*", new RMQNormalListener()); + String consumerGroup = initConsumerGroup(); + logger.info("use group: {}", consumerGroup); + RMQNormalConsumer consumer1 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer2 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer3 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); + RMQNormalConsumer consumer4 = getConsumer(nsAddr, consumerGroup, topic, "*", new RMQNormalListener()); TestUtils.waitForSeconds(waitTime); producer.send(msgSize); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java index 74a9e04ce2..df020c1121 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/PopSubCheckIT.java @@ -22,8 +22,9 @@ import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageRequestMode; import org.apache.rocketmq.logging.inner.Logger; import org.apache.rocketmq.test.base.BaseConf; -import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +import org.apache.rocketmq.test.client.rmq.RMQPopConsumer; +import org.apache.rocketmq.test.factory.ConsumerFactory; import org.apache.rocketmq.test.listener.rmq.concurrent.RMQNormalListener; import org.apache.rocketmq.test.util.RandomUtil; import org.apache.rocketmq.test.util.VerifyUtils; @@ -64,12 +65,14 @@ public class PopSubCheckIT extends BaseConf { RMQNormalProducer producer = getProducer(nsAddr, topic); producer.getProducer().setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); - RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener()); - for (String brokerAddr : new String[]{brokerController1.getBrokerAddr(), brokerController2.getBrokerAddr()}) { defaultMQAdminExt.setMessageRequestMode(brokerAddr, topic, group, MessageRequestMode.POP, 8, 60_000); } + RMQPopConsumer consumer = ConsumerFactory.getRMQPopConsumer(nsAddr, group, + topic, "*", new RMQNormalListener()); + mqClients.add(consumer); + int msgNum = 1; producer.send(msgNum); Assert.assertEquals("Not all sent succeeded", msgNum, producer.getAllUndupMsgBody().size()); @@ -80,7 +83,7 @@ public class PopSubCheckIT extends BaseConf { .containsExactlyElementsIn(producer.getAllMsgBody()); for (Object o : consumer.getListener().getAllOriginMsg()) { MessageClientExt msg = (MessageClientExt) o; - assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).isNotEmpty(); + assertThat(msg.getProperty(MessageConst.PROPERTY_POP_CK)).named("check pop meta").isNotEmpty(); } consumer.getListener().waitForMessageConsume(msgNum, 3_000 * 9); diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java index 564da5ceaf..0edbdbe2e3 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/MulTagSubIT.java @@ -92,7 +92,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubTwoTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -113,7 +113,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testSubThreeTabAndMatchTwo() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = String.format("%s||%s", tags[0], tags[1]); int msgSize = 10; @@ -135,7 +135,7 @@ public class MulTagSubIT extends BaseConf { @Test public void testNoMatch() { - String tags[] = {"jueyin1", "jueyin2", "jueyin3"}; + String[] tags = {"jueyin1", "jueyin2", "jueyin3"}; String subExpress = "no_match"; int msgSize = 10; diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java index 31321975b5..8de1b7d4e0 100644 --- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/tag/TagMessageWithMulConsumerIT.java @@ -84,7 +84,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSendMessagesWithTwoTag() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; int msgSize = 10; TagMessage tagMessage = new TagMessage(tags, topic, msgSize); @@ -113,7 +113,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testTwoConsumerOneMatchOneOtherMatchAll() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); int msgSize = 10; @@ -144,7 +144,7 @@ public class TagMessageWithMulConsumerIT extends BaseConf { @Test public void testSubKindsOf() { - String tags[] = {"jueyin1", "jueyin2"}; + String[] tags = {"jueyin1", "jueyin2"}; String sub1 = String.format("%s||%s", tags[0], tags[1]); String sub2 = String.format("%s|| noExist", tags[0]); String sub3 = tags[0]; diff --git a/tools/pom.xml b/tools/pom.xml index eeccae81bf..abe8197de3 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.8.1-SNAPSHOT + 4.9.1-SNAPSHOT 4.0.0 diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java index 6cdb960ccb..c27c85c66d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -129,12 +129,18 @@ public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + + return defaultMQAdminExtImpl.queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void start() throws MQClientException { defaultMQAdminExtImpl.start(); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java index 248f7abefb..8ae68cdecb 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -999,12 +999,18 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { } @Override - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, - long end) throws MQClientException, - InterruptedException { + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } + public QueryResult queryMessageByUniqKey(String topic, String key, int maxNum, long begin, + long end) throws MQClientException, InterruptedException { + + return this.mqClientInstance.getMQAdminImpl().queryMessageByUniqKey(topic, key, maxNum, begin, end); + } + @Override public void updateConsumeOffset(String brokerAddr, String consumeGroup, MessageQueue mq, long offset) throws RemotingException, InterruptedException, MQBrokerException { diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java index 5038123561..872a130d9f 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java @@ -38,7 +38,7 @@ import org.apache.rocketmq.tools.command.SubCommandException; public class CLusterSendMsgRTCommand implements SubCommand { - public static void main(String args[]) { + public static void main(String[] args) { } @Override diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java index 9ad075082f..8c7bec616d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.UtilAll; @@ -57,82 +58,43 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - public static void queryById(final DefaultMQAdminExt admin, final String topic, - final String msgId) throws MQClientException, - RemotingException, MQBrokerException, InterruptedException, IOException { - MessageExt msg = admin.viewMessage(topic, msgId); + public static void queryById(final DefaultMQAdminExt admin, final String topic, final String msgId, + final boolean showAll) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { - String bodyTmpFilePath = createBodyFile(msg); + QueryResult queryResult = admin.queryMessageByUniqKey(topic, msgId, 32, 0, Long.MAX_VALUE); + assert queryResult != null; + List list = queryResult.getMessageList(); + if (list == null || list.size() == 0) { + return; + } + list.sort((o1, o2) -> (int) (o1.getStoreTimestamp() - o2.getStoreTimestamp())); + for (int i = 0; i < (showAll ? list.size() : 1); i++) { + showMessage(admin, list.get(i), i); + } + } - System.out.printf("%-20s %s%n", - "Topic:", - msg.getTopic() - ); + private static void showMessage(final DefaultMQAdminExt admin, MessageExt msg, int index) throws IOException { + String bodyTmpFilePath = createBodyFile(msg, index); - System.out.printf("%-20s %s%n", - "Tags:", - "[" + msg.getTags() + "]" - ); + final String strFormat = "%-20s %s%n"; + final String intFormat = "%-20s %d%n"; - System.out.printf("%-20s %s%n", - "Keys:", - "[" + msg.getKeys() + "]" - ); - - System.out.printf("%-20s %d%n", - "Queue ID:", - msg.getQueueId() - ); - - System.out.printf("%-20s %d%n", - "Queue Offset:", - msg.getQueueOffset() - ); - - System.out.printf("%-20s %d%n", - "CommitLog Offset:", - msg.getCommitLogOffset() - ); - - System.out.printf("%-20s %d%n", - "Reconsume Times:", - msg.getReconsumeTimes() - ); - - System.out.printf("%-20s %s%n", - "Born Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getBornTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Store Timestamp:", - UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp()) - ); - - System.out.printf("%-20s %s%n", - "Born Host:", - RemotingHelper.parseSocketAddressAddr(msg.getBornHost()) - ); - - System.out.printf("%-20s %s%n", - "Store Host:", - RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()) - ); - - System.out.printf("%-20s %d%n", - "System Flag:", - msg.getSysFlag() - ); - - System.out.printf("%-20s %s%n", - "Properties:", - msg.getProperties() != null ? msg.getProperties().toString() : "" - ); - - System.out.printf("%-20s %s%n", - "Message Body Path:", - bodyTmpFilePath - ); + System.out.printf(strFormat, "Topic:", msg.getTopic()); + System.out.printf(strFormat, "Tags:", "[" + msg.getTags() + "]"); + System.out.printf(strFormat, "Keys:", "[" + msg.getKeys() + "]"); + System.out.printf(intFormat, "Queue ID:", msg.getQueueId()); + System.out.printf(intFormat, "Queue Offset:", msg.getQueueOffset()); + System.out.printf(intFormat, "CommitLog Offset:", msg.getCommitLogOffset()); + System.out.printf(intFormat, "Reconsume Times:", msg.getReconsumeTimes()); + System.out.printf(strFormat, "Born Timestamp:", UtilAll.timeMillisToHumanString2(msg.getBornTimestamp())); + System.out.printf(strFormat, "Store Timestamp:", UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp())); + System.out.printf(strFormat, "Born Host:", RemotingHelper.parseSocketAddressAddr(msg.getBornHost())); + System.out.printf(strFormat, "Store Host:", RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())); + System.out.printf(intFormat, "System Flag:", msg.getSysFlag()); + System.out.printf(strFormat, "Properties:", + msg.getProperties() != null ? msg.getProperties().toString() : ""); + System.out.printf(strFormat, "Message Body Path:", bodyTmpFilePath); try { List mtdList = admin.messageTrackDetail(msg); @@ -149,18 +111,21 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { } } - private static String createBodyFile(MessageExt msg) throws IOException { + private static String createBodyFile(MessageExt msg, int index) throws IOException { DataOutputStream dos = null; try { - String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; - File file = new File(bodyTmpFilePath); + StringBuffer bodyTmpFilePath = new StringBuffer("/tmp/rocketmq/msgbodys"); + File file = new File(bodyTmpFilePath.toString()); if (!file.exists()) { file.mkdirs(); } - bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); - dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + bodyTmpFilePath.append("/").append(msg.getMsgId()); + if (index > 0) { + bodyTmpFilePath.append("_" + index); + } + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath.toString())); dos.write(msg.getBody()); - return bodyTmpFilePath; + return bodyTmpFilePath.toString(); } finally { if (dos != null) dos.close(); @@ -195,6 +160,10 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("a", "showAll", false, "Print all message, the limit is 32"); + opt.setRequired(false); + options.addOption(opt); + return options; } @@ -202,11 +171,11 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { try { - defaultMQAdminExt = createMQAdminExt(rpcHook); final String msgId = commandLine.getOptionValue('i').trim(); final String topic = commandLine.getOptionValue('t').trim(); + final boolean showAll = commandLine.hasOption('a'); if (commandLine.hasOption('g') && commandLine.hasOption('d')) { final String consumerGroup = commandLine.getOptionValue('g').trim(); final String clientId = commandLine.getOptionValue('d').trim(); @@ -214,7 +183,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, topic, msgId); System.out.printf("%s", result); } else { - queryById(defaultMQAdminExt, topic, msgId); + queryById(defaultMQAdminExt, topic, msgId, showAll); } } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java index bed2763dad..7382ff568d 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java @@ -19,7 +19,6 @@ package org.apache.rocketmq.tools.command.message; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import org.apache.commons.codec.Charsets; import org.apache.commons.lang3.time.DateFormatUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.exception.MQClientException; @@ -38,7 +37,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; - public class QueryMsgTraceByIdSubCommand implements SubCommand { @Override @@ -46,6 +44,10 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { Option opt = new Option("i", "msgId", true, "Message Id"); opt.setRequired(true); options.addOption(opt); + + opt = new Option("t", "traceTopic", true, "The name value of message trace topic"); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -65,7 +67,11 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); try { final String msgId = commandLine.getOptionValue('i').trim(); - this.queryTraceByMsgId(defaultMQAdminExt, msgId); + String traceTopic = TopicValidator.RMQ_SYS_TRACE_TOPIC; + if (commandLine.hasOption('t')) { + traceTopic = commandLine.getOptionValue('t').trim(); + } + this.queryTraceByMsgId(defaultMQAdminExt, traceTopic, msgId); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + "command failed", e); } finally { @@ -73,14 +79,14 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { } } - private void queryTraceByMsgId(final DefaultMQAdminExt admin, String msgId) - throws MQClientException, InterruptedException { + private void queryTraceByMsgId(final DefaultMQAdminExt admin, String traceTopic, String msgId) + throws MQClientException, InterruptedException { admin.start(); - QueryResult queryResult = admin.queryMessage(TopicValidator.RMQ_SYS_TRACE_TOPIC, msgId, 64, 0, System.currentTimeMillis()); + QueryResult queryResult = admin.queryMessage(traceTopic, msgId, 64, 0, System.currentTimeMillis()); List messageList = queryResult.getMessageList(); List traceViews = new ArrayList<>(); for (MessageExt message : messageList) { - List traceView = TraceView.decodeFromTraceTransData(msgId, new String(message.getBody(), Charsets.UTF_8)); + List traceView = TraceView.decodeFromTraceTransData(msgId, message); traceViews.addAll(traceView); } @@ -92,20 +98,20 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { for (TraceView traceView : traceViews) { if (traceView.getMsgType().equals(TraceType.Pub.name())) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "#Type", - "#ProducerGroup", - "#ClientHost", - "#SendTime", - "#CostTimes", - "#Status" + "#Type", + "#ProducerGroup", + "#ClientHost", + "#SendTime", + "#CostTimes", + "#Status" ); System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "Pub", - traceView.getGroupName(), - traceView.getClientHost(), - DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), - traceView.getCostTime() + "ms", - traceView.getStatus() + "Pub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() ); System.out.printf("\n"); } @@ -124,22 +130,22 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { Iterator consumers = consumerTraceMap.keySet().iterator(); while (consumers.hasNext()) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "#Type", - "#ConsumerGroup", - "#ClientHost", - "#ConsumerTime", - "#CostTimes", - "#Status" + "#Type", + "#ConsumerGroup", + "#ClientHost", + "#ConsumerTime", + "#CostTimes", + "#Status" ); List consumerTraces = consumerTraceMap.get(consumers.next()); for (TraceView traceView : consumerTraces) { System.out.printf("%-10s %-20s %-20s %-20s %-10s %-10s%n", - "Sub", - traceView.getGroupName(), - traceView.getClientHost(), - DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), - traceView.getCostTime() + "ms", - traceView.getStatus() + "Sub", + traceView.getGroupName(), + traceView.getClientHost(), + DateFormatUtils.format(traceView.getTimeStamp(), "yyyy-MM-dd HH:mm:ss"), + traceView.getCostTime() + "ms", + traceView.getStatus() ); } System.out.printf("\n"); diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java index c770db01d4..a78a4a63ab 100644 --- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -19,14 +19,23 @@ package org.apache.rocketmq.tools.command.topic; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; import org.apache.rocketmq.tools.command.SubCommand; import org.apache.rocketmq.tools.command.SubCommandException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class TopicRouteSubCommand implements SubCommand { + private static final String FORMAT = "%-45s %-32s %-50s %-10s %-11s %-5s%n"; + @Override public String commandName() { return "topicRoute"; @@ -43,6 +52,9 @@ public class TopicRouteSubCommand implements SubCommand { opt.setRequired(true); options.addOption(opt); + opt = new Option("l", "list", false, "Use list format to print data"); + opt.setRequired(false); + options.addOption(opt); return options; } @@ -58,12 +70,46 @@ public class TopicRouteSubCommand implements SubCommand { String topic = commandLine.getOptionValue('t').trim(); TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); - String json = topicRouteData.toJson(true); - System.out.printf("%s%n", json); + printData(topicRouteData, commandLine.hasOption('l')); } catch (Exception e) { throw new SubCommandException(this.getClass().getSimpleName() + " command failed", e); } finally { defaultMQAdminExt.shutdown(); } } -} + + private void printData(TopicRouteData topicRouteData, boolean useListFormat) { + if (!useListFormat) { + System.out.printf("%s%n", topicRouteData.toJson(true)); + return; + } + + int totalReadQueue = 0, totalWriteQueue = 0; + List queueDataList = topicRouteData.getQueueDatas(); + Map map = new HashMap<>(); + for (QueueData queueData : queueDataList) { + map.put(queueData.getBrokerName(), queueData); + } + queueDataList.sort(Comparator.comparing(QueueData::getBrokerName)); + + List brokerDataList = topicRouteData.getBrokerDatas(); + brokerDataList.sort(Comparator.comparing(BrokerData::getBrokerName)); + + System.out.printf(FORMAT, "#ClusterName", "#BrokerName", "#BrokerAddrs", "#ReadQueue", "#WriteQueue", "#Perm"); + + for (BrokerData brokerData : brokerDataList) { + String brokerName = brokerData.getBrokerName(); + QueueData queueData = map.get(brokerName); + totalReadQueue += queueData.getReadQueueNums(); + totalWriteQueue += queueData.getWriteQueueNums(); + System.out.printf(FORMAT, brokerData.getCluster(), brokerName, brokerData.getBrokerAddrs(), + queueData.getReadQueueNums(), queueData.getWriteQueueNums(), queueData.getPerm()); + } + + for (int i = 0; i < 158; i++) { + System.out.print("-"); + } + System.out.printf("%n"); + System.out.printf(FORMAT, "Total:", map.keySet().size(), "", totalReadQueue, totalWriteQueue, ""); + } +} \ No newline at end of file