From 0bff61a545b59fea9f435f282768a89da8035e45 Mon Sep 17 00:00:00 2001 From: ZhangHao Date: Fri, 1 Mar 2019 09:43:30 +0800 Subject: [PATCH 01/73] [RIP-10] Add test cases for ConsumeMessageOrderlyService#consumeMessageDirectly --- .../ConsumeMessageOrderlyServiceTest.java | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java new file mode 100644 index 0000000000..4cfa011434 --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageOrderlyServiceTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.body.CMResult; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class ConsumeMessageOrderlyServiceTest { + private String consumerGroup; + private String topic = "FooBar"; + private String brokerName = "BrokerA"; + private DefaultMQPushConsumer pushConsumer; + + @Before + public void init() throws Exception { + consumerGroup = "FooBarGroup" + System.currentTimeMillis(); + pushConsumer = new DefaultMQPushConsumer(consumerGroup); + } + + @Test + public void testConsumeMessageDirectly_WithNoException() { + Map map = new HashMap(); + map.put(ConsumeOrderlyStatus.SUCCESS, CMResult.CR_SUCCESS); + map.put(ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT, CMResult.CR_LATER); + map.put(ConsumeOrderlyStatus.COMMIT, CMResult.CR_COMMIT); + map.put(ConsumeOrderlyStatus.ROLLBACK, CMResult.CR_ROLLBACK); + map.put(null, CMResult.CR_RETURN_NULL); + + for (ConsumeOrderlyStatus consumeOrderlyStatus : map.keySet()) { + final ConsumeOrderlyStatus status = consumeOrderlyStatus; + MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + return status; + } + }; + + ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly); + MessageExt msg = new MessageExt(); + msg.setTopic(topic); + assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(map.get(consumeOrderlyStatus))); + } + + } + + @Test + public void testConsumeMessageDirectly_WithException() { + MessageListenerOrderly listenerOrderly = new MessageListenerOrderly() { + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + throw new RuntimeException(); + } + }; + + ConsumeMessageOrderlyService consumeMessageOrderlyService = new ConsumeMessageOrderlyService(pushConsumer.getDefaultMQPushConsumerImpl(), listenerOrderly); + MessageExt msg = new MessageExt(); + msg.setTopic(topic); + assertTrue(consumeMessageOrderlyService.consumeMessageDirectly(msg, brokerName).getConsumeResult().equals(CMResult.CR_THROW_EXCEPTION)); + } + +} From c6c699ebf616c3e0199a9b71a7a441018b0e4757 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Tue, 27 Aug 2019 17:33:36 +0800 Subject: [PATCH 02/73] [RIP-16]impl rpc support --- .../rocketmq/broker/BrokerController.java | 28 ++ .../broker/client/ProducerManager.java | 10 + .../processor/ReplyMessageProcessor.java | 292 ++++++++++++++++++ .../processor/SendMessageProcessor.java | 6 +- .../broker/topic/TopicConfigManager.java | 8 + .../client/common/ClientErrorCode.java | 1 + .../exception/RequestTimeoutException.java | 40 +++ .../client/impl/ClientRemotingProcessor.java | 88 +++++- .../rocketmq/client/impl/MQClientAPIImpl.java | 22 +- .../impl/producer/DefaultMQProducerImpl.java | 258 ++++++++++++++++ .../client/producer/DefaultMQProducer.java | 106 +++++++ .../rocketmq/client/producer/MQProducer.java | 22 ++ .../client/producer/RequestCallback.java | 9 + .../client/producer/RequestFutureTable.java | 46 +++ .../producer/RequestResponseFuture.java | 117 +++++++ .../rocketmq/client/utils/MessageUtil.java | 30 ++ .../apache/rocketmq/common/BrokerConfig.java | 28 ++ .../org/apache/rocketmq/common/MixAll.java | 7 + .../rocketmq/common/message/MessageConst.java | 14 + .../rocketmq/common/protocol/RequestCode.java | 6 + .../header/ReplyMessageRequestHeader.java | 153 +++++++++ .../rocketmq/common/utils/RequestIdUtil.java | 9 + 22 files changed, 1290 insertions(+), 10 deletions(-) create mode 100644 broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java create mode 100644 common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a885cd0898..84b3e024ae 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -62,6 +62,7 @@ import org.apache.rocketmq.broker.processor.EndTransactionProcessor; import org.apache.rocketmq.broker.processor.PullMessageProcessor; import org.apache.rocketmq.broker.processor.QueryMessageProcessor; import org.apache.rocketmq.broker.processor.SendMessageProcessor; +import org.apache.rocketmq.broker.processor.ReplyMessageProcessor; import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; @@ -132,6 +133,7 @@ public class BrokerController { private final SlaveSynchronize slaveSynchronize; private final BlockingQueue sendThreadPoolQueue; private final BlockingQueue pullThreadPoolQueue; + private final BlockingQueue replyThreadPoolQueue; private final BlockingQueue queryThreadPoolQueue; private final BlockingQueue clientManagerThreadPoolQueue; private final BlockingQueue heartbeatThreadPoolQueue; @@ -147,6 +149,7 @@ public class BrokerController { private TopicConfigManager topicConfigManager; private ExecutorService sendMessageExecutor; private ExecutorService pullMessageExecutor; + private ExecutorService replyMessageExecutor; private ExecutorService queryMessageExecutor; private ExecutorService adminBrokerExecutor; private ExecutorService clientManageExecutor; @@ -194,6 +197,7 @@ public class BrokerController { this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); this.pullThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity()); + this.replyThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getReplyThreadPoolQueueCapacity()); this.queryThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getQueryThreadPoolQueueCapacity()); this.clientManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getClientManagerThreadPoolQueueCapacity()); this.consumerManagerThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getConsumerManagerThreadPoolQueueCapacity()); @@ -277,6 +281,14 @@ public class BrokerController { this.pullThreadPoolQueue, new ThreadFactoryImpl("PullMessageThread_")); + this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.replyThreadPoolQueue, + new ThreadFactoryImpl("ProcessReplyMessageThread_")); + this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( this.brokerConfig.getQueryMessageThreadPoolNums(), this.brokerConfig.getQueryMessageThreadPoolNums(), @@ -553,6 +565,18 @@ public class BrokerController { this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, this.pullMessageExecutor); this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + /** + * ReplyMessageProcessor + */ + ReplyMessageProcessor replyMessageProcessor = new ReplyMessageProcessor(this); + replyMessageProcessor.registerSendMessageHook(sendMessageHookList); + + this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor); + + /** * QueryMessageProcessor */ @@ -763,6 +787,10 @@ public class BrokerController { this.pullMessageExecutor.shutdown(); } + if (this.replyMessageExecutor != null) { + this.replyMessageExecutor.shutdown(); + } + if (this.adminBrokerExecutor != null) { this.adminBrokerExecutor.shutdown(); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 61ceae5386..db372fa8c8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -24,6 +24,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -43,6 +44,7 @@ public class ProducerManager { private final Lock groupChannelLock = new ReentrantLock(); private final HashMap> groupChannelTable = new HashMap>(); + private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); public ProducerManager() { } @@ -82,6 +84,7 @@ public class ProducerManager { long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp(); if (diff > CHANNEL_EXPIRED_TIMEOUT) { it.remove(); + clientChannelTable.remove(info.getClientId()); log.warn( "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); @@ -113,6 +116,7 @@ public class ProducerManager { final ClientChannelInfo clientChannelInfo = clientChannelInfoTable.remove(channel); if (clientChannelInfo != null) { + clientChannelTable.remove(clientChannelInfo.getClientId()); log.info( "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", clientChannelInfo.toString(), remoteAddr, group); @@ -146,6 +150,7 @@ public class ProducerManager { clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); if (null == clientChannelInfoFound) { channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); + clientChannelTable.put(clientChannelInfo.getClientId(), clientChannelInfo.getChannel()); log.info("new producer connected, group: {} channel: {}", group, clientChannelInfo.toString()); } @@ -171,6 +176,7 @@ public class ProducerManager { HashMap channelTable = this.groupChannelTable.get(group); if (null != channelTable && !channelTable.isEmpty()) { ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); + clientChannelTable.remove(clientChannelInfo.getClientId()); if (old != null) { log.info("unregister a producer[{}] from groupChannelTable {}", group, clientChannelInfo.toString()); @@ -223,4 +229,8 @@ public class ProducerManager { } return null; } + + public Channel findChannel(String clientId) { + return clientChannelTable.get(clientId); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java new file mode 100644 index 0000000000..00960d7990 --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -0,0 +1,292 @@ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.mqtrace.SendMessageContext; +import org.apache.rocketmq.common.*; +import org.apache.rocketmq.common.message.*; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.*; +import org.apache.rocketmq.store.stats.BrokerStatsManager; + +public class ReplyMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { + + public ReplyMessageProcessor(final BrokerController brokerController) { + super(brokerController); + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + SendMessageContext mqtraceContext = null; + SendMessageRequestHeader requestHeader = parseRequestHeader(request); + if (requestHeader == null) { + return null; + } + + mqtraceContext = buildMsgContext(ctx, requestHeader); + this.executeSendMessageHookBefore(ctx, request, mqtraceContext); + + RemotingCommand response = this.processReplyMessageRequest(ctx, request, mqtraceContext, requestHeader); + + this.executeSendMessageHookAfter(response, mqtraceContext); + return response; + } + + @Override + protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException { + SendMessageRequestHeaderV2 requestHeaderV2 = null; + SendMessageRequestHeader requestHeader = null; + switch (request.getCode()) { + case RequestCode.SEND_REPLY_MESSAGE_V2: + requestHeaderV2 = + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + case RequestCode.SEND_REPLY_MESSAGE: + if (null == requestHeaderV2) { + requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + } else { + requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); + } + default: + break; + } + return requestHeader; + } + + private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext ctx, + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) { + final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); + + response.setOpaque(request.getOpaque()); + + response.addExtField(MessageConst.PROPERTY_MSG_REGION, this.brokerController.getBrokerConfig().getRegionId()); + response.addExtField(MessageConst.PROPERTY_TRACE_SWITCH, String.valueOf(this.brokerController.getBrokerConfig().isTraceOn())); + + log.debug("receive SendReplyMessage request command, {}", request); + final long startTimstamp = this.brokerController.getBrokerConfig().getStartAcceptSendRequestTimeStamp(); + if (this.brokerController.getMessageStore().now() < startTimstamp) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("broker unable to service, until %s", UtilAll.timeMillisToHumanString2(startTimstamp))); + return response; + } + + response.setCode(-1); + super.msgCheck(ctx, requestHeader, response); + if (response.getCode() != -1) { + return response; + } + + final byte[] body = request.getBody(); + + int queueIdInt = requestHeader.getQueueId(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + + if (queueIdInt < 0) { + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(requestHeader.getTopic()); + msgInner.setQueueId(queueIdInt); + msgInner.setBody(body); + msgInner.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); + msgInner.setPropertiesString(requestHeader.getProperties()); + msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + + boolean pushOk = this.pushReplyMessage(ctx, requestHeader, msgInner, response); + + if (pushOk && this.brokerController.getBrokerConfig().isStoreReplyMessageEnable()) { + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + this.handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); + } else { + responseHeader.setMsgId("0"); + responseHeader.setQueueId(0); + responseHeader.setQueueOffset(0L); + } + + return response; + } + + private boolean pushReplyMessage(final ChannelHandlerContext ctx, final SendMessageRequestHeader requestHeader, final Message msg, final RemotingCommand response) { + ReplyMessageRequestHeader replyMessageRequestHeader = new ReplyMessageRequestHeader(); + replyMessageRequestHeader.setBornHost(ctx.channel().remoteAddress().toString()); + replyMessageRequestHeader.setStoreHost(this.getStoreHost().toString()); + replyMessageRequestHeader.setStoreTimestamp(System.currentTimeMillis()); + replyMessageRequestHeader.setProducerGroup(requestHeader.getProducerGroup()); + replyMessageRequestHeader.setTopic(requestHeader.getTopic()); + replyMessageRequestHeader.setDefaultTopic(requestHeader.getDefaultTopic()); + replyMessageRequestHeader.setDefaultTopicQueueNums(requestHeader.getDefaultTopicQueueNums()); + replyMessageRequestHeader.setQueueId(requestHeader.getQueueId()); + replyMessageRequestHeader.setSysFlag(requestHeader.getSysFlag()); + replyMessageRequestHeader.setBornTimestamp(requestHeader.getBornTimestamp()); + replyMessageRequestHeader.setFlag(requestHeader.getFlag()); + replyMessageRequestHeader.setProperties(requestHeader.getProperties()); + replyMessageRequestHeader.setReconsumeTimes(requestHeader.getReconsumeTimes()); + replyMessageRequestHeader.setUnitMode(requestHeader.isUnitMode()); + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, replyMessageRequestHeader); + request.setBody(msg.getBody()); + + String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + boolean pushOk = false; + + if (senderId != null) { + Channel channel = this.brokerController.getProducerManager().findChannel(senderId); + if (channel != null) { + msg.getProperties().put(MessageConst.PROPERTY_PUSH_REPLY_TIME, String.valueOf(System.currentTimeMillis())); + replyMessageRequestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + + try { + RemotingCommand pushResponse = this.brokerController.getBroker2Client().callClient(channel, request); + assert pushResponse != null; + switch (pushResponse.getCode()) { + case ResponseCode.SUCCESS: { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + pushOk = true; + break; + } + default: { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("push reply message to requester fail"); + log.warn("push reply message to <{}> return fail, remark: {}", senderId, response.getRemark()); + } + } + } catch (InterruptedException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("push reply message to requester fail"); + log.warn("push reply message to <{}> fail. {}", senderId, channel, e); + } catch (RemotingException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("push reply message to requester fail"); + log.warn("push reply message to <{}> fail. {}", senderId, channel, e); + } + } else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("channel of <" + senderId + "> not found"); + log.warn("push reply message fial, channel of <{}> not found.", senderId); + } + return pushOk; + } + log.warn("REPLY_TO is null, can not reply message"); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("REPLY_TO is null"); + return pushOk; + } + + private void handlePutMessageResult(PutMessageResult putMessageResult, final RemotingCommand response, + final RemotingCommand request, final MessageExt msg, + final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, + int queueIdInt) { + if (putMessageResult == null) { + response.setRemark("push reply to requester success, but store putMessage return null"); + return ; + } + boolean sendOK = false; + + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + sendOK = true; + response.setCode(ResponseCode.SUCCESS); + break; + case FLUSH_DISK_TIMEOUT: + response.setCode(ResponseCode.FLUSH_DISK_TIMEOUT); + sendOK = true; + break; + case FLUSH_SLAVE_TIMEOUT: + response.setCode(ResponseCode.FLUSH_SLAVE_TIMEOUT); + sendOK = true; + break; + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + sendOK = true; + break; + + // Failed + case CREATE_MAPEDFILE_FAILED: +// response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("create mapped file failed, server is busy or broken."); + break; + case MESSAGE_ILLEGAL: + case PROPERTIES_SIZE_EXCEEDED: +// response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark( + "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + break; + case SERVICE_NOT_AVAILABLE: +// response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark( + "service not available now, maybe disk full, maybe your broker machine memory too small."); + break; + case OS_PAGECACHE_BUSY: +// response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); + break; + case UNKNOWN_ERROR: +// response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: +// response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + + String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); + if (sendOK) { + this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); + this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + this.brokerController.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); + response.setRemark(null); + responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + + if (hasSendMessageHook()) { + sendMessageContext.setMsgId(responseHeader.getMsgId()); + sendMessageContext.setQueueId(responseHeader.getQueueId()); + sendMessageContext.setQueueOffset(responseHeader.getQueueOffset()); + + int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); + int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + } else { + if (hasSendMessageHook()) { + int wroteSize = request.getBody().length; + int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + + sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); + sendMessageContext.setCommercialSendTimes(incValue); + sendMessageContext.setCommercialSendSize(wroteSize); + sendMessageContext.setCommercialOwner(owner); + } + } + } +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java index 8035ae6f18..2589a7547d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/SendMessageProcessor.java @@ -343,11 +343,13 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement msgInner.setBody(body); msgInner.setFlag(requestHeader.getFlag()); MessageAccessor.setProperties(msgInner, MessageDecoder.string2messageProperties(requestHeader.getProperties())); - msgInner.setPropertiesString(requestHeader.getProperties()); msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); msgInner.setBornHost(ctx.channel().remoteAddress()); msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_CLUSTER, clusterName); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); PutMessageResult putMessageResult = null; Map oriProps = MessageDecoder.string2messageProperties(requestHeader.getProperties()); String traFlag = oriProps.get(MessageConst.PROPERTY_TRANSACTION_PREPARED); @@ -536,6 +538,8 @@ public class SendMessageProcessor extends AbstractSendMessageProcessor implement messageExtBatch.setBornHost(ctx.channel().remoteAddress()); messageExtBatch.setStoreHost(this.getStoreHost()); messageExtBatch.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + String clusterName = this.brokerController.getBrokerConfig().getBrokerClusterName(); + MessageAccessor.putProperty(messageExtBatch, MessageConst.PROPERTY_CLUSTER, clusterName); PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessages(messageExtBatch); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 8f215cdcb9..577fff57ee 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -134,6 +134,14 @@ public class TopicConfigManager extends ConfigManager { this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); } } + { + String topic = this.brokerController.getBrokerConfig().getBrokerClusterName() + "_" + MixAll.REPLY_TOPIC_POSTFIX; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } } public boolean isSystemTopic(final String topic) { diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java index 62a95dfa5a..633670aecc 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java @@ -23,4 +23,5 @@ public class ClientErrorCode { public static final int BROKER_NOT_EXIST_EXCEPTION = 10003; public static final int NO_NAME_SERVER_EXCEPTION = 10004; public static final int NOT_FOUND_TOPIC_EXCEPTION = 10005; + public static final int REQUEST_TIMEOUT_EXCEPTION = 10006; } \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java new file mode 100644 index 0000000000..fd338d6004 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java @@ -0,0 +1,40 @@ +package org.apache.rocketmq.client.exception; + +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.help.FAQUrl; + +public class RequestTimeoutException extends Exception { + private static final long serialVersionUID = -5758410930844185841L; + private int responseCode; + private String errorMessage; + + public RequestTimeoutException(String errorMessage, Throwable cause) { + super(errorMessage, cause); + this.responseCode = -1; + this.errorMessage = errorMessage; + } + + public RequestTimeoutException(int responseCode, String errorMessage) { + super("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + public int getResponseCode() { + return responseCode; + } + + public RequestTimeoutException setResponseCode(final int responseCode) { + this.responseCode = responseCode; + return this; + } + + public String getErrorMessage() { + return errorMessage; + } + + public void setErrorMessage(final String errorMessage) { + this.errorMessage = errorMessage; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 0bd810a1e1..7a17a04307 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -16,6 +16,8 @@ */ package org.apache.rocketmq.client.impl; +import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -25,11 +27,10 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.MQProducerInner; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.RequestFutureTable; +import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.message.*; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; @@ -43,6 +44,8 @@ import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestH import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -76,6 +79,9 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { case RequestCode.CONSUME_MESSAGE_DIRECTLY: return this.consumeMessageDirectly(ctx, request); + + case RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT: + return this.receiveReplyMssage(ctx, request); default: break; } @@ -213,4 +219,78 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { return response; } + + private RemotingCommand receiveReplyMssage(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + long receiveTime = System.currentTimeMillis(); + ReplyMessageRequestHeader requestHeader = (ReplyMessageRequestHeader) request.decodeCommandCustomHeader(ReplyMessageRequestHeader.class); + + try { + MessageExt msg = new MessageExt(); + msg.setTopic(requestHeader.getTopic()); + msg.setQueueId(requestHeader.getQueueId()); + msg.setStoreTimestamp(requestHeader.getStoreTimestamp()); + + if (requestHeader.getBornHost() != null) { + String[] bornHostArr = requestHeader.getBornHost().split("/"); + String bornHost/*ip:port*/ = bornHostArr[bornHostArr.length - 1]; + String[] host = bornHost.split(":"); + if (host.length == 2) + msg.setBornHost(new InetSocketAddress(host[0], Integer.parseInt(host[1]))); + } + + if (requestHeader.getStoreHost() != null) { + String[] storeHostArr = requestHeader.getStoreHost().split("/"); + String storeHost = storeHostArr[storeHostArr.length - 1]; + String[] host = storeHost.split(":"); + if (host.length == 2) + msg.setStoreHost(new InetSocketAddress(host[0], Integer.parseInt(host[1]))); + } + + byte[] body = request.getBody(); + if ((requestHeader.getSysFlag() & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + try { + body = UtilAll.uncompress(body); + } catch (IOException e) { + log.warn("err when uncompress constant", e); + } + } + msg.setBody(body); + msg.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msg, MessageDecoder.string2messageProperties(requestHeader.getProperties())); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REPLY_MESSAGE_ARRIVE_TIME, String.valueOf(receiveTime)); + msg.setBornTimestamp(requestHeader.getBornTimestamp()); + msg.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); + log.debug("receive reply message :{}", msg); + + processReplyMessage(msg); + } catch (Exception e) { + log.warn("unknown err when receiveRRReplyMsg", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("process reply message fail"); + } + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + private void processReplyMessage(MessageExt replyMsg) { + final String uniqueId = replyMsg.getUserProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final RequestResponseFuture requestResponseFuture = RequestFutureTable.getRequestFutureTable().get(uniqueId); + if (requestResponseFuture != null) { + requestResponseFuture.putResponseMessage(replyMsg); + + RequestFutureTable.getRequestFutureTable().remove(uniqueId); + + if (requestResponseFuture.getRequestCallback() != null) { + requestResponseFuture.getRequestCallback().onSuccess(replyMsg); + } else { + requestResponseFuture.putResponseMessage(replyMsg); + } + } else { + log.warn(String.format("receive reply message, but not matched any request, REQUEST_UNIQ_ID: %s", uniqueId)); + } + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index b743af93d4..f780adf9de 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -199,6 +199,8 @@ public class MQClientAPIImpl { this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_RUNNING_INFO, this.clientRemotingProcessor, null); this.remotingClient.registerProcessor(RequestCode.CONSUME_MESSAGE_DIRECTLY, this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, this.clientRemotingProcessor, null); } public List getNameServerAddressList() { @@ -419,13 +421,23 @@ public class MQClientAPIImpl { ) throws RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); RemotingCommand request = null; - if (sendSmartMsg || msg instanceof MessageBatch) { - SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); - request = RemotingCommand.createRequestCommand(msg instanceof MessageBatch ? RequestCode.SEND_BATCH_MESSAGE : RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + String msgType = msg.getProperty(MessageConst.PROPERTY_MESSAGE_TYPE); + boolean isReply = msgType != null && msgType.equals(MixAll.REPLY_MESSAGE_FLAG); + if (isReply) { + if (sendSmartMsg) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE_V2, requestHeaderV2); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_REPLY_MESSAGE, requestHeader); + } } else { - request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + if (sendSmartMsg || msg instanceof MessageBatch) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(msg instanceof MessageBatch ? RequestCode.SEND_BATCH_MESSAGE : RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + } else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + } } - request.setBody(msg.getBody()); switch (communicationMode) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 62aaef3b13..7a0bc4cd98 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -24,6 +24,8 @@ import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -39,6 +41,7 @@ import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.CheckForbiddenContext; import org.apache.rocketmq.client.hook.CheckForbiddenHook; import org.apache.rocketmq.client.hook.SendMessageContext; @@ -52,6 +55,9 @@ import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; import org.apache.rocketmq.client.producer.MessageQueueSelector; +import org.apache.rocketmq.client.producer.RequestCallback; +import org.apache.rocketmq.client.producer.RequestFutureTable; +import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.client.producer.SendCallback; import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.producer.SendStatus; @@ -79,11 +85,13 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHe import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.utils.RequestIdUtil; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; @@ -107,6 +115,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { private final BlockingQueue asyncSenderThreadPoolQueue; private final ExecutorService defaultAsyncSenderExecutor; private ExecutorService asyncSenderExecutor; + private final Timer timer = new Timer("RequestHouseKeepingService", true); public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { this(defaultMQProducer, null); @@ -212,6 +221,17 @@ public class DefaultMQProducerImpl implements MQProducerInner { } this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + + this.timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + try { + RequestFutureTable.scanExpiredRequest(); + } catch (Throwable e) { + log.error("scan RequestFutureTable exception", e); + } + } + }, 1000 * 3, 1000); } private void checkConfig() throws MQClientException { @@ -1310,6 +1330,244 @@ public class DefaultMQProducerImpl implements MQProducerInner { return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); } + public Message request(Message msg, long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, timeout); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + if (responseMessage == null) { + if (requestResponseFuture.isSendReqeustOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + } + } + + public void request(Message msg, final RequestCallback requestCallback, long timeout) throws RemotingException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, timeout); + } catch (Exception ex) { + log.warn("send request message to <{}> failed.", msg.getTopic(), ex); + throw new RemotingSendRequestException(msg.getTopic(), ex); + } + } + + public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, timeout); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + if (responseMessage == null) { + if (requestResponseFuture.isSendReqeustOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + } + } + + public void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, final long timeout) throws RemotingException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, timeout); + } catch (Exception ex) { + log.warn("send request message to <{}> failed.", msg.getTopic(), ex); + throw new RemotingSendRequestException(msg.getTopic(), ex); + } + } + + public Message request(final Message msg, final MessageQueue mq, final long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setSendReqeustOk(false); + requestResponseFuture.putResponseMessage(null); + requestResponseFuture.setCause(e); + } + }, null, timeout); + + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + if (responseMessage == null) { + if (requestResponseFuture.isSendReqeustOk()) { + throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + } else { + throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); + } + } + return responseMessage; + } finally { + RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + } + } + + public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws RemotingException { + prepareSendRequest(msg, timeout); + final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + + try { + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, null, timeout); + } catch (Exception ex) { + log.warn("send request message to <{}> failed.", msg.getTopic(), ex); + throw new RemotingSendRequestException(msg.getTopic(), ex); + } + } + + private void requestFail(final String requestUniqId) { + RequestResponseFuture responseFuture = RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + if (responseFuture != null) { + responseFuture.setSendReqeustOk(false); + responseFuture.putResponseMessage(null); + try { + responseFuture.executeRequestCallback(); + } catch (Exception e) { + log.warn("execute requestCallback in requestFail, and callback throw", e); + } + } + } + + private void prepareSendRequest(final Message msg, long timeout){ + String requestUniqId = RequestIdUtil.createUniqueRequestId(); + String requestClientId = this.getmQClientFactory().getClientId(); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO, requestClientId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TTL, String.valueOf(timeout)); + + boolean hasRouteData = this.getmQClientFactory().getTopicRouteTable().containsKey(msg.getTopic()); + if (!hasRouteData) { + long beginTimestamp = System.currentTimeMillis(); + this.tryToFindTopicPublishInfo(msg.getTopic()); + this.getmQClientFactory().sendHeartbeatToAllBrokerWithLock(); + long cost = System.currentTimeMillis() - beginTimestamp; + if (cost > 500) { + log.warn("prepare send request for <{}> cost {} ms", msg.getTopic(), cost); + } + } + } + + private SendResult reply(final Message msg, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeoutMillis); + } + + private SendResult reply(final Message msg, final SendCallback sendCallback, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + return this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeoutMillis); + } + + public void replyOneway(final Message msg) throws RemotingException, MQClientException, InterruptedException { + this.sendOneway(msg); + } + public ConcurrentMap getTopicPublishInfoTable() { return topicPublishInfoTable; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index b4acf8f1c3..4789c4d33e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -24,6 +24,7 @@ import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; @@ -582,6 +583,111 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback, timeout); } + /** + * + * @param msg + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + */ + @Override + public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, + RemotingException, MQBrokerException, InterruptedException { + msg.setTopic(withNamespace(msg.getTopic())); + return this.defaultMQProducerImpl.request(msg, timeout); + } + + /** + * + * @param msg + * @param requestCallback + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Override + public void request(final Message msg, final RequestCallback requestCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException, MQBrokerException, RequestTimeoutException { + this.defaultMQProducerImpl.request(msg, requestCallback, timeout); + } + + /** + * + * @param msg + * @param selector + * @param arg + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + */ + @Override + public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { + return this.defaultMQProducerImpl.request(msg, selector, arg, timeout); + } + + /** + * + * @param msg + * @param selector + * @param arg + * @param requestCallback + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Override + public void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, + InterruptedException { + this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout); + } + + /** + * + * @param msg + * @param mq + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + */ + @Override + public Message request(final Message msg, final MessageQueue mq, final long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + return this.defaultMQProducerImpl.request(msg, mq, timeout); + } + + /** + * + * @param msg + * @param mq + * @param requestCallback + * @param timeout + * @return + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Override + public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout); + } + /** * Same to {@link #sendOneway(Message)} with message queue selector specified. * diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index 1af6005748..ce8d4a5024 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.rocketmq.client.MQAdmin; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -98,4 +99,25 @@ public interface MQProducer extends MQAdmin { SendResult send(final Collection msgs, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + + //for rpc + Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, + RemotingException, MQBrokerException, InterruptedException; + + void request(final Message msg, final RequestCallback requestCallback, final long timeout) + throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException, MQBrokerException; + + Message request(final Message msg, final MessageQueueSelector selector, final Object arg, + final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + void request(final Message msg, final MessageQueueSelector selector, final Object arg, + final RequestCallback requestCallback, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, + InterruptedException; + + Message request(final Message msg, final MessageQueue mq, final long timeout) + throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; + + void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) + throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java new file mode 100644 index 0000000000..4a1874f929 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java @@ -0,0 +1,9 @@ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.message.Message; + +public interface RequestCallback { + void onSuccess(final Message message); + + void onException(final Throwable e); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java new file mode 100644 index 0000000000..74195be33b --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java @@ -0,0 +1,46 @@ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.RequestTimeoutException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class RequestFutureTable { + private static InternalLogger log = ClientLogger.getLog(); + private static ConcurrentHashMap requestFutureTable = new ConcurrentHashMap(); + + public static ConcurrentHashMap getRequestFutureTable(){ + return requestFutureTable; + } + + public static void scanExpiredRequest(){ + final List rfList = new LinkedList(); + Iterator> it = requestFutureTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + RequestResponseFuture rep = next.getValue(); + + if (rep.isTimeout()) { + it.remove(); + rfList.add(rep); + log.warn("remove timeout request, REQUEST_UNIQ_ID={}" + rep.getRequestUniqId()); + } + } + + for (RequestResponseFuture rf : rfList) { + try { + Throwable cause = new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, "request timeout, no reply message."); + rf.setCause(cause); + rf.executeRequestCallback(); + } catch (Throwable e) { + log.warn("scanResponseTable, operationComplete Exception", e); + } + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java new file mode 100644 index 0000000000..a5c5feebdb --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -0,0 +1,117 @@ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.common.message.Message; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class RequestResponseFuture { + private final String requestUniqId; + private long timeoutMillis; + private final RequestCallback requestCallback; + private final long beginTimestamp = System.currentTimeMillis(); + private CountDownLatch countDownLatch = new CountDownLatch(1); + + private AtomicBoolean ececuteCallbackOnlyOnce = new AtomicBoolean(false); + private volatile Message responseMsg = null; + private volatile boolean sendReqeustOk = true; + private volatile Throwable cause = null; + private final Message requestMsg = null; + + + public RequestResponseFuture(String requestUniqId, long timeoutMillis, RequestCallback requestCallback){ + this.requestUniqId = requestUniqId; + this.timeoutMillis = timeoutMillis; + this.requestCallback = requestCallback; + } + + public void executeRequestCallback(){ + if (requestCallback != null) { + if (sendReqeustOk && cause == null) { + requestCallback.onSuccess(responseMsg); + } else { + requestCallback.onException(cause); + } + } + } + + public boolean isTimeout(){ + long diff = System.currentTimeMillis() - this.beginTimestamp; + return diff > this.timeoutMillis; + } + + public Message waitResponseMessage(final long timeout) throws InterruptedException { + this.countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + return this.responseMsg; + } + + public void putResponseMessage(final Message responseMsg){ + this.responseMsg = responseMsg; + this.countDownLatch.countDown(); + } + + public String getRequestUniqId() { + return requestUniqId; + } + + public long getTimeoutMillis() { + return timeoutMillis; + } + + public void setTimeoutMillis(long timeoutMillis) { + this.timeoutMillis = timeoutMillis; + } + + public RequestCallback getRequestCallback() { + return requestCallback; + } + + public long getBeginTimestamp() { + return beginTimestamp; + } + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + public AtomicBoolean getEcecuteCallbackOnlyOnce() { + return ececuteCallbackOnlyOnce; + } + + public void setEcecuteCallbackOnlyOnce(AtomicBoolean ececuteCallbackOnlyOnce) { + this.ececuteCallbackOnlyOnce = ececuteCallbackOnlyOnce; + } + + public Message getResponseMsg() { + return responseMsg; + } + + public void setResponseMsg(Message responseMsg) { + this.responseMsg = responseMsg; + } + + public boolean isSendReqeustOk() { + return sendReqeustOk; + } + + public void setSendReqeustOk(boolean sendReqeustOk) { + this.sendReqeustOk = sendReqeustOk; + } + + public Message getRequestMsg() { + return requestMsg; + } + + public Throwable getCause() { + return cause; + } + + public void setCause(Throwable cause) { + this.cause = cause; + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java new file mode 100644 index 0000000000..ce41775227 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -0,0 +1,30 @@ +package org.apache.rocketmq.client.utils; + +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; + +public class MessageUtil { + public static Message createReplyMessage(final Message requestMessage) { + if (requestMessage != null) { + Message replyMessage = new Message(); + String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); + String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + String requestUniqId = requestMessage.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); + if (cluster == null) { + + } + String replyTopic = MixAll.getReplyTopic(cluster); + replyMessage.setTopic(replyTopic); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO, replyTo); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); + + return replyMessage; + } + return null; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index 1c3f37d00d..c3613e7d4c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -61,6 +61,7 @@ public class BrokerConfig { */ private int sendMessageThreadPoolNums = 1; //16 + Runtime.getRuntime().availableProcessors() * 4; private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; + private int processReplyMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; private int queryMessageThreadPoolNums = 8 + Runtime.getRuntime().availableProcessors(); private int adminBrokerThreadPoolNums = 16; @@ -83,6 +84,7 @@ public class BrokerConfig { private boolean fetchNamesrvAddrByAddressServer = false; private int sendThreadPoolQueueCapacity = 10000; private int pullThreadPoolQueueCapacity = 100000; + private int replyThreadPoolQueueCapacity = 10000; private int queryThreadPoolQueueCapacity = 20000; private int clientManagerThreadPoolQueueCapacity = 1000000; private int consumerManagerThreadPoolQueueCapacity = 1000000; @@ -180,6 +182,8 @@ public class BrokerConfig { @ImportantField private boolean aclEnable = false; + private boolean storeReplyMessageEnable = true; + public static String localHostName() { try { return InetAddress.getLocalHost().getHostName(); @@ -374,6 +378,14 @@ public class BrokerConfig { this.pullMessageThreadPoolNums = pullMessageThreadPoolNums; } + public int getProcessReplyMessageThreadPoolNums() { + return processReplyMessageThreadPoolNums; + } + + public void setProcessReplyMessageThreadPoolNums(int processReplyMessageThreadPoolNums) { + this.processReplyMessageThreadPoolNums = processReplyMessageThreadPoolNums; + } + public int getQueryMessageThreadPoolNums() { return queryMessageThreadPoolNums; } @@ -470,6 +482,14 @@ public class BrokerConfig { this.pullThreadPoolQueueCapacity = pullThreadPoolQueueCapacity; } + public int getReplyThreadPoolQueueCapacity() { + return replyThreadPoolQueueCapacity; + } + + public void setReplyThreadPoolQueueCapacity(int replyThreadPoolQueueCapacity) { + this.replyThreadPoolQueueCapacity = replyThreadPoolQueueCapacity; + } + public int getQueryThreadPoolQueueCapacity() { return queryThreadPoolQueueCapacity; } @@ -765,4 +785,12 @@ public class BrokerConfig { public void setAclEnable(boolean aclEnable) { this.aclEnable = aclEnable; } + + public boolean isStoreReplyMessageEnable() { + return storeReplyMessageEnable; + } + + public void setStoreReplyMessageEnable(boolean storeReplyMessageEnable) { + this.storeReplyMessageEnable = storeReplyMessageEnable; + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index 0af65dff0f..a6151e55f7 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -84,6 +84,7 @@ public class MixAll { public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; + public static final String REPLY_TOPIC_POSTFIX = "REPLY_TOPIC"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String UNIQUE_MSG_QUERY_FLAG = "_UNIQUE_KEY_QUERY"; public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; @@ -96,6 +97,8 @@ public class MixAll { public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; + public static final String REPLY_MESSAGE_FLAG = "reply"; + public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); String wsDomainSubgroup = System.getProperty("rocketmq.namesrv.domain.subgroup", "nsaddr"); @@ -110,6 +113,10 @@ public class MixAll { return RETRY_GROUP_TOPIC_PREFIX + consumerGroup; } + public static String getReplyTopic(final String clusterName) { + return clusterName + "_" + REPLY_TOPIC_POSTFIX; + } + public static boolean isSysConsumerGroup(final String consumerGroup) { return consumerGroup.startsWith(CID_RMQ_SYS_PREFIX); } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index aa8481643a..178e417c7a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -45,6 +45,13 @@ public class MessageConst { public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String PROPERTY_INSTANCE_ID = "INSTANCE_ID"; + public static final String PROPERTY_REQUEST_UNIQ_ID = "REQUEST_UNIQ_ID"; + public static final String PROPERTY_MESSAGE_REPLY_TO = "REPLY_TO"; + public static final String PROPERTY_MESSAGE_TTL = "TTL"; + public static final String PROPERTY_REPLY_MESSAGE_ARRIVE_TIME = "ARRIVE_TIME"; + public static final String PROPERTY_PUSH_REPLY_TIME = "PUSH_REPLY_TIME"; + public static final String PROPERTY_CLUSTER = "CLUSTER"; + public static final String PROPERTY_MESSAGE_TYPE = "MSG_TYPE"; public static final String KEY_SEPARATOR = " "; @@ -74,5 +81,12 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES); STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); STRING_HASH_SET.add(PROPERTY_INSTANCE_ID); + STRING_HASH_SET.add(PROPERTY_REQUEST_UNIQ_ID); + STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO); + STRING_HASH_SET.add(PROPERTY_MESSAGE_TTL); + STRING_HASH_SET.add(PROPERTY_REPLY_MESSAGE_ARRIVE_TIME); + STRING_HASH_SET.add(PROPERTY_PUSH_REPLY_TIME); + STRING_HASH_SET.add(PROPERTY_CLUSTER); + STRING_HASH_SET.add(PROPERTY_MESSAGE_TYPE); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java index 58c4b9fe90..ef7f2a1f4d 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/RequestCode.java @@ -180,4 +180,10 @@ public class RequestCode { * resume logic of checking half messages that have been put in TRANS_CHECK_MAXTIME_TOPIC before */ public static final int RESUME_CHECK_HALF_MESSAGE = 323; + + public static final int SEND_REPLY_MESSAGE = 324; + + public static final int SEND_REPLY_MESSAGE_V2 = 325; + + public static final int PUSH_REPLY_MESSAGE_TO_CLIENT = 326; } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java new file mode 100644 index 0000000000..465baeae86 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java @@ -0,0 +1,153 @@ +package org.apache.rocketmq.common.protocol.header; + +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.annotation.CFNotNull; +import org.apache.rocketmq.remoting.annotation.CFNullable; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; + +public class ReplyMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer defaultTopicQueueNums; + @CFNotNull + private Integer queueId; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long bornTimestamp; + @CFNotNull + private Integer flag; + @CFNullable + private String properties; + @CFNullable + private Integer reconsumeTimes; + @CFNullable + private boolean unitMode = false; + + @CFNotNull + private String bornHost; + @CFNotNull + private String storeHost; + @CFNotNull + private long storeTimestamp; + + public void checkFields() throws RemotingCommandException { + } + + public String getProducerGroup() { + return producerGroup; + } + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + public String getTopic() { + return topic; + } + + public void setTopic(String topic) { + this.topic = topic; + } + + public String getDefaultTopic() { + return defaultTopic; + } + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + public Integer getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + public Integer getQueueId() { + return queueId; + } + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + public Integer getSysFlag() { + return sysFlag; + } + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + public Long getBornTimestamp() { + return bornTimestamp; + } + + public void setBornTimestamp(Long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + public Integer getFlag() { + return flag; + } + + public void setFlag(Integer flag) { + this.flag = flag; + } + + public String getProperties() { + return properties; + } + + public void setProperties(String properties) { + this.properties = properties; + } + + public Integer getReconsumeTimes() { + return reconsumeTimes; + } + + public void setReconsumeTimes(Integer reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + public boolean isUnitMode() { + return unitMode; + } + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + public String getBornHost() { + return bornHost; + } + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } + + public String getStoreHost() { + return storeHost; + } + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + public long getStoreTimestamp() { + return storeTimestamp; + } + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java new file mode 100644 index 0000000000..9a58e5fdb6 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java @@ -0,0 +1,9 @@ +package org.apache.rocketmq.common.utils; + +import java.util.UUID; + +public class RequestIdUtil { + public static String createUniqueRequestId(){ + return UUID.randomUUID().toString(); + } +} From 46b2e1250b7d728b3a2212624603e785c578bdf1 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Tue, 27 Aug 2019 21:16:31 +0800 Subject: [PATCH 03/73] [RIP-16]impl rpc support - code format --- .../rocketmq/broker/BrokerController.java | 37 ++++------ .../broker/client/ProducerManager.java | 3 +- .../processor/ReplyMessageProcessor.java | 70 ++++++++++++------ .../broker/topic/TopicConfigManager.java | 4 +- .../exception/RequestTimeoutException.java | 20 +++++- .../client/impl/ClientRemotingProcessor.java | 14 ++-- .../rocketmq/client/impl/MQClientAPIImpl.java | 12 ++-- .../impl/producer/DefaultMQProducerImpl.java | 69 +++++++++--------- .../client/producer/DefaultMQProducer.java | 72 +++++++------------ .../rocketmq/client/producer/MQProducer.java | 17 ++--- .../client/producer/RequestCallback.java | 17 +++++ .../client/producer/RequestFutureTable.java | 30 ++++++-- .../producer/RequestResponseFuture.java | 34 ++++++--- .../rocketmq/client/utils/MessageUtil.java | 17 +++++ .../apache/rocketmq/common/BrokerConfig.java | 2 +- .../org/apache/rocketmq/common/MixAll.java | 8 +-- .../header/ReplyMessageRequestHeader.java | 17 +++++ .../rocketmq/common/utils/RequestIdUtil.java | 19 ++++- 18 files changed, 284 insertions(+), 178 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 84b3e024ae..8739d9fc62 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -61,8 +61,8 @@ import org.apache.rocketmq.broker.processor.ConsumerManageProcessor; import org.apache.rocketmq.broker.processor.EndTransactionProcessor; import org.apache.rocketmq.broker.processor.PullMessageProcessor; import org.apache.rocketmq.broker.processor.QueryMessageProcessor; -import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.broker.processor.ReplyMessageProcessor; +import org.apache.rocketmq.broker.processor.SendMessageProcessor; import org.apache.rocketmq.broker.slave.SlaveSynchronize; import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; import org.apache.rocketmq.broker.topic.TopicConfigManager; @@ -166,7 +166,7 @@ public class BrokerController { private TransactionalMessageService transactionalMessageService; private AbstractTransactionalMessageCheckListener transactionalMessageCheckListener; private Future slaveSyncFuture; - private Map accessValidatorMap = new HashMap(); + private Map accessValidatorMap = new HashMap(); public BrokerController( final BrokerConfig brokerConfig, @@ -245,7 +245,7 @@ public class BrokerController { this.brokerConfig); if (messageStoreConfig.isEnableDLegerCommitLog()) { DLedgerRoleChangeHandler roleChangeHandler = new DLedgerRoleChangeHandler(this, (DefaultMessageStore) messageStore); - ((DLedgerCommitLog)((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); + ((DLedgerCommitLog) ((DefaultMessageStore) messageStore).getCommitLog()).getdLedgerServer().getdLedgerLeaderElector().addRoleChangeHandler(roleChangeHandler); } this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); //load plugin @@ -282,12 +282,12 @@ public class BrokerController { new ThreadFactoryImpl("PullMessageThread_")); this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( - this.brokerConfig.getProcessReplyMessageThreadPoolNums(), - this.brokerConfig.getProcessReplyMessageThreadPoolNums(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.replyThreadPoolQueue, - new ThreadFactoryImpl("ProcessReplyMessageThread_")); + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + this.brokerConfig.getProcessReplyMessageThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.replyThreadPoolQueue, + new ThreadFactoryImpl("ProcessReplyMessageThread_")); this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( this.brokerConfig.getQueryMessageThreadPoolNums(), @@ -513,9 +513,9 @@ public class BrokerController { return; } - for (AccessValidator accessValidator: accessValidators) { + for (AccessValidator accessValidator : accessValidators) { final AccessValidator validator = accessValidator; - accessValidatorMap.put(validator.getClass(),validator); + accessValidatorMap.put(validator.getClass(), validator); this.registerServerRPCHook(new RPCHook() { @Override @@ -531,14 +531,13 @@ public class BrokerController { } } - private void initialRpcHooks() { List rpcHooks = ServiceProvider.load(ServiceProvider.RPC_HOOK_ID, RPCHook.class); if (rpcHooks == null || rpcHooks.isEmpty()) { return; } - for (RPCHook rpcHook: rpcHooks) { + for (RPCHook rpcHook : rpcHooks) { this.registerServerRPCHook(rpcHook); } } @@ -576,7 +575,6 @@ public class BrokerController { this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE, replyMessageProcessor, replyMessageExecutor); this.fastRemotingServer.registerProcessor(RequestCode.SEND_REPLY_MESSAGE_V2, replyMessageProcessor, replyMessageExecutor); - /** * QueryMessageProcessor */ @@ -887,8 +885,6 @@ public class BrokerController { handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); } - - this.registerBrokerAll(true, false, true); this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @@ -911,7 +907,6 @@ public class BrokerController { this.brokerFastFailure.start(); } - } public synchronized void registerIncrementBrokerData(TopicConfig topicConfig, DataVersion dataVersion) { @@ -1125,7 +1120,6 @@ public class BrokerController { this.transactionalMessageCheckListener = transactionalMessageCheckListener; } - public BlockingQueue getEndTransactionThreadPoolQueue() { return endTransactionThreadPoolQueue; @@ -1146,8 +1140,7 @@ public class BrokerController { public void run() { try { BrokerController.this.slaveSynchronize.syncAll(); - } - catch (Throwable e) { + } catch (Throwable e) { log.error("ScheduledTask SlaveSynchronize syncAll error.", e); } } @@ -1193,8 +1186,6 @@ public class BrokerController { log.info("Finish to change to slave brokerName={} brokerId={}", brokerConfig.getBrokerName(), brokerId); } - - public void changeToMaster(BrokerRole role) { if (role == BrokerRole.SLAVE) { return; @@ -1244,6 +1235,4 @@ public class BrokerController { } } - - } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index db372fa8c8..bd007b0531 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.broker.client; import io.netty.channel.Channel; - import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -28,7 +27,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; - import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; @@ -46,6 +44,7 @@ public class ProducerManager { new HashMap>(); private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); + public ProducerManager() { } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index 00960d7990..23e1e9ac8b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -1,11 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.broker.processor; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; -import org.apache.rocketmq.common.*; -import org.apache.rocketmq.common.message.*; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; @@ -16,7 +38,8 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.store.*; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.stats.BrokerStatsManager; public class ReplyMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { @@ -26,7 +49,8 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen } @Override - public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { + public RemotingCommand processRequest(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { SendMessageContext mqtraceContext = null; SendMessageRequestHeader requestHeader = parseRequestHeader(request); if (requestHeader == null) { @@ -49,13 +73,13 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen switch (request.getCode()) { case RequestCode.SEND_REPLY_MESSAGE_V2: requestHeaderV2 = - (SendMessageRequestHeaderV2) request - .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); case RequestCode.SEND_REPLY_MESSAGE: if (null == requestHeaderV2) { requestHeader = - (SendMessageRequestHeader) request - .decodeCommandCustomHeader(SendMessageRequestHeader.class); + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); } else { requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); } @@ -66,11 +90,11 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen } private RemotingCommand processReplyMessageRequest(final ChannelHandlerContext ctx, - final RemotingCommand request, - final SendMessageContext sendMessageContext, - final SendMessageRequestHeader requestHeader) { + final RemotingCommand request, + final SendMessageContext sendMessageContext, + final SendMessageRequestHeader requestHeader) { final RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); - final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader)response.readCustomHeader(); + final SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) response.readCustomHeader(); response.setOpaque(request.getOpaque()); @@ -126,7 +150,8 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen return response; } - private boolean pushReplyMessage(final ChannelHandlerContext ctx, final SendMessageRequestHeader requestHeader, final Message msg, final RemotingCommand response) { + private boolean pushReplyMessage(final ChannelHandlerContext ctx, final SendMessageRequestHeader requestHeader, + final Message msg, final RemotingCommand response) { ReplyMessageRequestHeader replyMessageRequestHeader = new ReplyMessageRequestHeader(); replyMessageRequestHeader.setBornHost(ctx.channel().remoteAddress().toString()); replyMessageRequestHeader.setStoreHost(this.getStoreHost().toString()); @@ -194,12 +219,13 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen } private void handlePutMessageResult(PutMessageResult putMessageResult, final RemotingCommand response, - final RemotingCommand request, final MessageExt msg, - final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, ChannelHandlerContext ctx, - int queueIdInt) { + final RemotingCommand request, final MessageExt msg, + final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, + ChannelHandlerContext ctx, + int queueIdInt) { if (putMessageResult == null) { response.setRemark("push reply to requester success, but store putMessage return null"); - return ; + return; } boolean sendOK = false; @@ -231,12 +257,12 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen case PROPERTIES_SIZE_EXCEEDED: // response.setCode(ResponseCode.MESSAGE_ILLEGAL); response.setRemark( - "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); break; case SERVICE_NOT_AVAILABLE: // response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); response.setRemark( - "service not available now, maybe disk full, maybe your broker machine memory too small."); + "service not available now, maybe disk full, maybe your broker machine memory too small."); break; case OS_PAGECACHE_BUSY: // response.setCode(ResponseCode.SYSTEM_ERROR); @@ -256,7 +282,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen if (sendOK) { this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(), - putMessageResult.getAppendMessageResult().getWroteBytes()); + putMessageResult.getAppendMessageResult().getWroteBytes()); this.brokerController.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); response.setRemark(null); responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); @@ -270,7 +296,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen int commercialBaseCount = brokerController.getBrokerConfig().getCommercialBaseCount(); int wroteSize = putMessageResult.getAppendMessageResult().getWroteBytes(); - int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT) * commercialBaseCount; sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_SUCCESS); sendMessageContext.setCommercialSendTimes(incValue); @@ -280,7 +306,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen } else { if (hasSendMessageHook()) { int wroteSize = request.getBody().length; - int incValue = (int)Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); + int incValue = (int) Math.ceil(wroteSize / BrokerStatsManager.SIZE_PER_COUNT); sendMessageContext.setCommercialSendStats(BrokerStatsManager.StatsType.SEND_FAILURE); sendMessageContext.setCommercialSendTimes(incValue); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java index 577fff57ee..cb2901117f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java @@ -34,11 +34,11 @@ import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.common.protocol.body.KVTable; import org.apache.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; public class TopicConfigManager extends ConfigManager { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java index fd338d6004..2d756ece61 100644 --- a/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java +++ b/client/src/main/java/org/apache/rocketmq/client/exception/RequestTimeoutException.java @@ -1,7 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.client.exception; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.help.FAQUrl; public class RequestTimeoutException extends Exception { private static final long serialVersionUID = -5758410930844185841L; @@ -16,7 +32,7 @@ public class RequestTimeoutException extends Exception { public RequestTimeoutException(int responseCode, String errorMessage) { super("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " - + errorMessage); + + errorMessage); this.responseCode = responseCode; this.errorMessage = errorMessage; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 7a17a04307..e9072df3a6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -16,13 +16,12 @@ */ package org.apache.rocketmq.client.impl; +import io.netty.channel.ChannelHandlerContext; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; - -import io.netty.channel.ChannelHandlerContext; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.impl.producer.MQProducerInner; @@ -30,7 +29,11 @@ import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.RequestFutureTable; import org.apache.rocketmq.client.producer.RequestResponseFuture; import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.message.*; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; @@ -43,8 +46,8 @@ import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRe import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; import org.apache.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; -import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -52,7 +55,6 @@ import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; - public class ClientRemotingProcessor implements NettyRequestProcessor { private final InternalLogger log = ClientLogger.getLog(); private final MQClientInstance mqClientFactory; @@ -221,7 +223,7 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { } private RemotingCommand receiveReplyMssage(ChannelHandlerContext ctx, - RemotingCommand request) throws RemotingCommandException { + RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); long receiveTime = System.currentTimeMillis(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index f780adf9de..d478d7410d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -27,7 +27,6 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.PullCallback; @@ -304,8 +303,8 @@ public class MQClientAPIImpl { requestHeader.setDefaultGroupPerm(plainAccessConfig.getDefaultGroupPerm()); requestHeader.setDefaultTopicPerm(plainAccessConfig.getDefaultTopicPerm()); requestHeader.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress()); - requestHeader.setTopicPerms(UtilAll.List2String(plainAccessConfig.getTopicPerms(),",")); - requestHeader.setGroupPerms(UtilAll.List2String(plainAccessConfig.getGroupPerms(),",")); + requestHeader.setTopicPerms(UtilAll.List2String(plainAccessConfig.getTopicPerms(), ",")); + requestHeader.setGroupPerms(UtilAll.List2String(plainAccessConfig.getGroupPerms(), ",")); RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_ACL_CONFIG, requestHeader); @@ -344,7 +343,7 @@ public class MQClientAPIImpl { throw new MQClientException(response.getCode(), response.getRemark()); } - public void updateGlobalWhiteAddrsConfig(final String addr, final String globalWhiteAddrs,final long timeoutMillis) + public void updateGlobalWhiteAddrsConfig(final String addr, final String globalWhiteAddrs, final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { UpdateGlobalWhiteAddrsConfigRequestHeader requestHeader = new UpdateGlobalWhiteAddrsConfigRequestHeader(); @@ -366,7 +365,8 @@ public class MQClientAPIImpl { throw new MQClientException(response.getCode(), response.getRemark()); } - public ClusterAclVersionInfo getBrokerClusterAclInfo(final String addr,final long timeoutMillis) throws RemotingCommandException, InterruptedException, RemotingTimeoutException, + public ClusterAclVersionInfo getBrokerClusterAclInfo(final String addr, + final long timeoutMillis) throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, MQBrokerException { RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_ACL_INFO, null); @@ -389,7 +389,7 @@ public class MQClientAPIImpl { } throw new MQBrokerException(response.getCode(), response.getRemark()); - + } public SendResult sendMessage( diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 7a0bc4cd98..6e15ed1a09 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -103,19 +103,17 @@ public class DefaultMQProducerImpl implements MQProducerInner { new ConcurrentHashMap(); private final ArrayList sendMessageHookList = new ArrayList(); private final RPCHook rpcHook; + private final BlockingQueue asyncSenderThreadPoolQueue; + private final ExecutorService defaultAsyncSenderExecutor; + private final Timer timer = new Timer("RequestHouseKeepingService", true); protected BlockingQueue checkRequestQueue; protected ExecutorService checkExecutor; private ServiceState serviceState = ServiceState.CREATE_JUST; private MQClientInstance mQClientFactory; private ArrayList checkForbiddenHookList = new ArrayList(); private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); - private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); - - private final BlockingQueue asyncSenderThreadPoolQueue; - private final ExecutorService defaultAsyncSenderExecutor; private ExecutorService asyncSenderExecutor; - private final Timer timer = new Timer("RequestHouseKeepingService", true); public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { this(defaultMQProducer, null); @@ -291,6 +289,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { /** * This method will be removed in the version 5.0.0 and getCheckListener is recommended. + * * @return */ @Override @@ -484,13 +483,14 @@ public class DefaultMQProducerImpl implements MQProducerInner { * DEFAULT ASYNC ------------------------------------------------------- */ public void send(Message msg, - SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param sendCallback * @param timeout the sendCallback will be invoked at most time @@ -525,7 +525,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { } - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); } @@ -701,11 +700,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { } private SendResult sendKernelImpl(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + final MessageQueue mq, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { long beginStartTime = System.currentTimeMillis(); String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); if (null == brokerAddr) { @@ -1010,8 +1009,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param mq * @param sendCallback @@ -1137,8 +1137,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { } /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. - * A new one will be provided in next version + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * * @param msg * @param selector * @param arg @@ -1149,7 +1150,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { * @throws InterruptedException */ @Deprecated - public void send(final Message msg, final MessageQueueSelector selector, final Object arg, final SendCallback sendCallback, final long timeout) + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { final long beginStartTime = System.currentTimeMillis(); ExecutorService executor = this.getAsyncSenderExecutor(); @@ -1193,7 +1195,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter localTransactionExecuter, final Object arg) + final LocalTransactionExecuter localTransactionExecuter, final Object arg) throws MQClientException { TransactionListener transactionListener = getCheckListener(); if (null == localTransactionExecuter && null == transactionListener) { @@ -1330,7 +1332,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); } - public Message request(Message msg, long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { + public Message request(Message msg, + long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1356,7 +1359,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { if (responseMessage == null) { if (requestResponseFuture.isSendReqeustOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, - "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); } @@ -1394,8 +1397,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, - InterruptedException, RequestTimeoutException { + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1421,7 +1424,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { if (responseMessage == null) { if (requestResponseFuture.isSendReqeustOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, - "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); } @@ -1433,7 +1436,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void request(final Message msg, final MessageQueueSelector selector, final Object arg, - final RequestCallback requestCallback, final long timeout) throws RemotingException { + final RequestCallback requestCallback, final long timeout) throws RemotingException { prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1460,7 +1463,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public Message request(final Message msg, final MessageQueue mq, final long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1486,7 +1489,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { if (responseMessage == null) { if (requestResponseFuture.isSendReqeustOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, - "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); + "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { throw new MQClientException("send request message to <" + msg.getTopic() + "> fail", requestResponseFuture.getCause()); } @@ -1498,7 +1501,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws RemotingException { + throws RemotingException { prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1537,7 +1540,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } - private void prepareSendRequest(final Message msg, long timeout){ + private void prepareSendRequest(final Message msg, long timeout) { String requestUniqId = RequestIdUtil.createUniqueRequestId(); String requestClientId = this.getmQClientFactory().getClientId(); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); @@ -1556,11 +1559,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } - private SendResult reply(final Message msg, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + private SendResult reply(final Message msg, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeoutMillis); } - private SendResult reply(final Message msg, final SendCallback sendCallback, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + private SendResult reply(final Message msg, final SendCallback sendCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { return this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeoutMillis); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index 4789c4d33e..bb017ef212 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -43,38 +43,29 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; /** - * This class is the entry point for applications intending to send messages. - *

+ * This class is the entry point for applications intending to send messages.

* * It's fine to tune fields which exposes getter/setter methods, but keep in mind, all of them should work well out of - * box for most scenarios. - *

+ * box for most scenarios.

* * This class aggregates various send methods to deliver messages to brokers. Each of them has pros and - * cons; you'd better understand strengths and weakness of them before actually coding. - *

+ * cons; you'd better understand strengths and weakness of them before actually coding.

* - *

- * Thread Safety: After configuring and starting process, this class can be regarded as thread-safe - * and used among multiple threads context. - *

+ *

Thread Safety: After configuring and starting process, this class can be regarded as thread-safe + * and used among multiple threads context.

*/ public class DefaultMQProducer extends ClientConfig implements MQProducer { - private final InternalLogger log = ClientLogger.getLog(); - /** * Wrapping internal implementations for virtually all methods presented in this class. */ protected final transient DefaultMQProducerImpl defaultMQProducerImpl; - + private final InternalLogger log = ClientLogger.getLog(); /** * Producer group conceptually aggregates all producer instances of exactly same role, which is particularly - * important when transactional messages are involved. - *

+ * important when transactional messages are involved.

* - * For non-transactional messages, it does not matter as long as it's unique per process. - *

+ * For non-transactional messages, it does not matter as long as it's unique per process.

* * See {@linktourl http://rocketmq.apache.org/docs/core-concept/} for more discussion. */ @@ -101,16 +92,14 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { private int compressMsgBodyOverHowmuch = 1024 * 4; /** - * Maximum number of retry to perform internally before claiming sending failure in synchronous mode. - *

+ * Maximum number of retry to perform internally before claiming sending failure in synchronous mode.

* * This may potentially cause message duplication which is up to application developers to resolve. */ private int retryTimesWhenSendFailed = 2; /** - * Maximum number of retry to perform internally before claiming sending failure in asynchronous mode. - *

+ * Maximum number of retry to perform internally before claiming sending failure in asynchronous mode.

* * This may potentially cause message duplication which is up to application developers to resolve. */ @@ -269,14 +258,10 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Start this producer instance. - *

+ * Start this producer instance.

* - * - * Much internal initializing procedures are carried out to make this instance prepared, thus, it's a must to invoke - * this method before sending or querying messages. - * - *

+ * Much internal initializing procedures are carried out to make this instance prepared, thus, it's a must + * to invoke this method before sending or querying messages.

* * @throws MQClientException if there is any unexpected error. */ @@ -317,8 +302,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Send message in synchronous mode. This method returns only when the sending procedure totally completes. - *

+ * Send message in synchronous mode. This method returns only when the sending procedure totally completes.

* * Warn: this method has internal retry-mechanism, that is, internal implementation will retry * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may potentially @@ -360,11 +344,9 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * Send message to broker asynchronously. - *

+ * Send message to broker asynchronously.

* - * This method returns immediately. On sending completion, sendCallback will be executed. - *

+ * This method returns immediately. On sending completion, sendCallback will be executed.

* * Similar to {@link #send(Message)}, internal implementation would potentially retry up to {@link * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and @@ -584,7 +566,6 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * * @param msg * @param timeout * @return @@ -595,13 +576,12 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, - RemotingException, MQBrokerException, InterruptedException { + RemotingException, MQBrokerException, InterruptedException { msg.setTopic(withNamespace(msg.getTopic())); return this.defaultMQProducerImpl.request(msg, timeout); } /** - * * @param msg * @param requestCallback * @param timeout @@ -612,12 +592,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public void request(final Message msg, final RequestCallback requestCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException, MQBrokerException, RequestTimeoutException { + throws MQClientException, RemotingException, InterruptedException, MQBrokerException, RequestTimeoutException { this.defaultMQProducerImpl.request(msg, requestCallback, timeout); } /** - * * @param msg * @param selector * @param arg @@ -630,13 +609,12 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, - InterruptedException, RequestTimeoutException { + final long timeout) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException, RequestTimeoutException { return this.defaultMQProducerImpl.request(msg, selector, arg, timeout); } /** - * * @param msg * @param selector * @param arg @@ -649,13 +627,12 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public void request(final Message msg, final MessageQueueSelector selector, final Object arg, - final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, - InterruptedException { + final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, + InterruptedException { this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout); } /** - * * @param msg * @param mq * @param timeout @@ -667,12 +644,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public Message request(final Message msg, final MessageQueue mq, final long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { return this.defaultMQProducerImpl.request(msg, mq, timeout); } /** - * * @param msg * @param mq * @param requestCallback @@ -684,7 +660,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { */ @Override public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws MQClientException, RemotingException, InterruptedException { + throws MQClientException, RemotingException, InterruptedException { this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index ce8d4a5024..a837c3d232 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -102,22 +102,23 @@ public interface MQProducer extends MQAdmin { //for rpc Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, - RemotingException, MQBrokerException, InterruptedException; + RemotingException, MQBrokerException, InterruptedException; void request(final Message msg, final RequestCallback requestCallback, final long timeout) - throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException, MQBrokerException; + throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException, MQBrokerException; Message request(final Message msg, final MessageQueueSelector selector, final Object arg, - final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, - InterruptedException; + final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, + InterruptedException; void request(final Message msg, final MessageQueueSelector selector, final Object arg, - final RequestCallback requestCallback, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, - InterruptedException; + final RequestCallback requestCallback, + final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, + InterruptedException; Message request(final Message msg, final MessageQueue mq, final long timeout) - throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; + throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException; + throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java index 4a1874f929..3107ba57d6 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestCallback.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.client.producer; import org.apache.rocketmq.common.message.Message; diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java index 74195be33b..6ce53119ea 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java @@ -1,25 +1,41 @@ -package org.apache.rocketmq.client.producer; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import org.apache.rocketmq.client.common.ClientErrorCode; -import org.apache.rocketmq.client.exception.RequestTimeoutException; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.logging.InternalLogger; +package org.apache.rocketmq.client.producer; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.RequestTimeoutException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.logging.InternalLogger; public class RequestFutureTable { private static InternalLogger log = ClientLogger.getLog(); private static ConcurrentHashMap requestFutureTable = new ConcurrentHashMap(); - public static ConcurrentHashMap getRequestFutureTable(){ + public static ConcurrentHashMap getRequestFutureTable() { return requestFutureTable; } - public static void scanExpiredRequest(){ + public static void scanExpiredRequest() { final List rfList = new LinkedList(); Iterator> it = requestFutureTable.entrySet().iterator(); while (it.hasNext()) { diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java index a5c5feebdb..d65b479434 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -1,32 +1,46 @@ -package org.apache.rocketmq.client.producer; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import org.apache.rocketmq.common.message.Message; +package org.apache.rocketmq.client.producer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.rocketmq.common.message.Message; public class RequestResponseFuture { private final String requestUniqId; - private long timeoutMillis; private final RequestCallback requestCallback; private final long beginTimestamp = System.currentTimeMillis(); + private final Message requestMsg = null; + private long timeoutMillis; private CountDownLatch countDownLatch = new CountDownLatch(1); - private AtomicBoolean ececuteCallbackOnlyOnce = new AtomicBoolean(false); private volatile Message responseMsg = null; private volatile boolean sendReqeustOk = true; private volatile Throwable cause = null; - private final Message requestMsg = null; - - public RequestResponseFuture(String requestUniqId, long timeoutMillis, RequestCallback requestCallback){ + public RequestResponseFuture(String requestUniqId, long timeoutMillis, RequestCallback requestCallback) { this.requestUniqId = requestUniqId; this.timeoutMillis = timeoutMillis; this.requestCallback = requestCallback; } - public void executeRequestCallback(){ + public void executeRequestCallback() { if (requestCallback != null) { if (sendReqeustOk && cause == null) { requestCallback.onSuccess(responseMsg); @@ -36,7 +50,7 @@ public class RequestResponseFuture { } } - public boolean isTimeout(){ + public boolean isTimeout() { long diff = System.currentTimeMillis() - this.beginTimestamp; return diff > this.timeoutMillis; } @@ -46,7 +60,7 @@ public class RequestResponseFuture { return this.responseMsg; } - public void putResponseMessage(final Message responseMsg){ + public void putResponseMessage(final Message responseMsg) { this.responseMsg = responseMsg; this.countDownLatch.countDown(); } diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index ce41775227..62dd36dd61 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.client.utils; import org.apache.rocketmq.common.MixAll; diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java index c3613e7d4c..a7568f0a20 100644 --- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java @@ -769,7 +769,7 @@ public class BrokerConfig { public void setMsgTraceTopicName(String msgTraceTopicName) { this.msgTraceTopicName = msgTraceTopicName; } - + public boolean isTraceTopicEnable() { return traceTopicEnable; } diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index a6151e55f7..0f7f0aa7ab 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -45,8 +45,6 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; public class MixAll { - private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); - public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; public static final String ROCKETMQ_HOME_PROPERTY = "rocketmq.home.dir"; public static final String NAMESRV_ADDR_ENV = "NAMESRV_ADDR"; @@ -74,30 +72,26 @@ public class MixAll { public static final String CID_ONSAPI_OWNER_GROUP = "CID_ONSAPI_OWNER"; public static final String CID_ONSAPI_PULL_GROUP = "CID_ONSAPI_PULL"; public static final String CID_RMQ_SYS_PREFIX = "CID_RMQ_SYS_"; - public static final List LOCAL_INET_ADDRESS = getLocalInetAddress(); public static final String LOCALHOST = localhost(); public static final String DEFAULT_CHARSET = "UTF-8"; public static final long MASTER_ID = 0L; public static final long CURRENT_JVM_PID = getPID(); - public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; - public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; public static final String REPLY_TOPIC_POSTFIX = "REPLY_TOPIC"; public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; public static final String UNIQUE_MSG_QUERY_FLAG = "_UNIQUE_KEY_QUERY"; public static final String DEFAULT_TRACE_REGION_ID = "DefaultRegion"; public static final String CONSUME_CONTEXT_TYPE = "ConsumeContextType"; - public static final String RMQ_SYS_TRANS_HALF_TOPIC = "RMQ_SYS_TRANS_HALF_TOPIC"; public static final String RMQ_SYS_TRACE_TOPIC = "RMQ_SYS_TRACE_TOPIC"; public static final String RMQ_SYS_TRANS_OP_HALF_TOPIC = "RMQ_SYS_TRANS_OP_HALF_TOPIC"; public static final String TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; public static final String CID_SYS_RMQ_TRANS = "CID_RMQ_SYS_TRANS"; public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; - public static final String REPLY_MESSAGE_FLAG = "reply"; + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); public static String getWSAddr() { String wsDomainName = System.getProperty("rocketmq.namesrv.domain", DEFAULT_NAMESRV_ADDR_LOOKUP); diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java index 465baeae86..3bb09073f7 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ReplyMessageRequestHeader.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.common.protocol.header; import org.apache.rocketmq.remoting.CommandCustomHeader; diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java index 9a58e5fdb6..86c5096df5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java @@ -1,9 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.rocketmq.common.utils; import java.util.UUID; public class RequestIdUtil { - public static String createUniqueRequestId(){ + public static String createUniqueRequestId() { return UUID.randomUUID().toString(); } } From 1d3a5083a41bb7a6b53fa93b17cd63f9c0625726 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 28 Aug 2019 16:18:41 +0800 Subject: [PATCH 04/73] [RIP-16]fix test case error --- .../org/apache/rocketmq/broker/client/ProducerManagerTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index 08dbb9c754..c47a0cb7c6 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -20,6 +20,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelFuture; import java.lang.reflect.Field; import java.util.HashMap; +import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,7 +43,7 @@ public class ProducerManagerTest { @Before public void init() { producerManager = new ProducerManager(); - clientInfo = new ClientChannelInfo(channel); + clientInfo = new ClientChannelInfo(channel, "clientId", LanguageCode.JAVA, 0); } @Test From 7e221e51e898a0d4f73d854409fa8b97f7778408 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 11 Sep 2019 19:40:04 +0800 Subject: [PATCH 05/73] add example of request-response model --- .../rocketmq/client/utils/MessageUtil.java | 22 ++--- .../example/rpc/AsyncRequestProducer.java | 64 ++++++++++++++ .../rocketmq/example/rpc/RequestProducer.java | 54 ++++++++++++ .../example/rpc/ResponseConsumer.java | 83 +++++++++++++++++++ 4 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java create mode 100644 example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java create mode 100644 example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index 62dd36dd61..81da669ff8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -17,31 +17,31 @@ package org.apache.rocketmq.client.utils; +import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; public class MessageUtil { - public static Message createReplyMessage(final Message requestMessage) { + public static Message createReplyMessage(final Message requestMessage) throws MQClientException { if (requestMessage != null) { Message replyMessage = new Message(); String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); String requestUniqId = requestMessage.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); - if (cluster == null) { + if (cluster != null) { + String replyTopic = MixAll.getReplyTopic(cluster); + replyMessage.setTopic(replyTopic); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO, replyTo); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); + return replyMessage; } - String replyTopic = MixAll.getReplyTopic(cluster); - replyMessage.setTopic(replyTopic); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO, replyTo); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); - - return replyMessage; } - return null; + throw new MQClientException(-1, "create reply message fail."); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java new file mode 100644 index 0000000000..d2475c165b --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.client.producer.RequestCallback; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class AsyncRequestProducer { + private static final InternalLogger log = ClientLogger.getLog(); + + public static void main(String[] args) throws MQClientException, InterruptedException { + String producerGroup = "ProducerGroup-Name"; + String namesrvAddr = "10.255.2.37:9876;10.255.2.37:9875"; + String topic = "RequestTopic"; + long ttl = 3000; + + DefaultMQProducer producer = new DefaultMQProducer(producerGroup); + producer.setNamesrvAddr(namesrvAddr); + producer.start(); + + try { + Message msg = new Message(topic, + "", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + long begin = System.currentTimeMillis(); + producer.request(msg, new RequestCallback() { + @Override + public void onSuccess(Message message) { + long cost = System.currentTimeMillis() - begin; + System.out.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, message); + } + + @Override + public void onException(Throwable e) { + System.err.printf("request to <%s> fail.", topic); + } + }, ttl); + } catch (Exception e) { + log.warn("", e); + } +// producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java new file mode 100644 index 0000000000..73397738d5 --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.DefaultMQProducer; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.common.RemotingHelper; + +public class RequestProducer { + private static final InternalLogger log = ClientLogger.getLog(); + + public static void main(String[] args) throws MQClientException, InterruptedException { + String producerGroup = "ProducerGroup-Name"; + String namesrvAddr = "10.255.2.37:9876;10.255.2.37:9875"; + String topic = "RequestTopic"; + long ttl = 3000; + + DefaultMQProducer producer = new DefaultMQProducer(producerGroup); + producer.setNamesrvAddr(namesrvAddr); + producer.start(); + + try { + Message msg = new Message(topic, + "", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + + long begin = System.currentTimeMillis(); + Message retMsg = producer.request(msg, ttl); + long cost = System.currentTimeMillis() - begin; + System.err.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); + } catch (Exception e) { + log.warn("", e); + } + producer.shutdown(); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java new file mode 100644 index 0000000000..ff3652fe3a --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.example.rpc; + +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.utils.MessageUtil; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.exception.RemotingException; + +public class ResponseConsumer { + private static final InternalLogger log = ClientLogger.getLog(); + + public static void main(String[] args) throws InterruptedException, MQClientException { + String consumerGroup = "ConsumeGroup-Name"; + String namesrvAddr = "10.255.2.37:9873"; + String topic = "RequestTopic"; + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup); + consumer.setNamesrvAddr(namesrvAddr); + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, ConsumeConcurrentlyContext context) { + System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); + for (MessageExt msg : msgs) { + try { + log.info("handle message: {} body={}", msg, new String(msg.getBody())); + String replyTo = msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + + //You must use MessageUtil to creage reply message, otherwise reply message maybe wrong. + Message replyMessage = MessageUtil.createReplyMessage(msg); + replyMessage.setBody("reply message contents.".getBytes()); + + //maybe you should create a producer to send reply message. + SendResult sendResult = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(replyMessage, 3000); + System.out.printf("reply msg %s to %s , %s %n", replyMessage.toString(), replyTo, sendResult.toString()); + } catch (MQClientException e) { + e.printStackTrace(); + } catch (RemotingException e) { + e.printStackTrace(); + } catch (MQBrokerException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.subscribe(topic, "*"); + consumer.start(); + System.out.printf("Consumer Started.%n"); + } +} From cc0a5e391ea90e37936b7ba63b8c273f8cbcc660 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 11 Sep 2019 19:47:47 +0800 Subject: [PATCH 06/73] code format --- .../rocketmq/example/rpc/RequestProducer.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java index 73397738d5..a83f1db87f 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java @@ -38,14 +38,14 @@ public class RequestProducer { producer.start(); try { - Message msg = new Message(topic, - "", - "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); + Message msg = new Message(topic, + "", + "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); - long begin = System.currentTimeMillis(); - Message retMsg = producer.request(msg, ttl); - long cost = System.currentTimeMillis() - begin; - System.err.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); + long begin = System.currentTimeMillis(); + Message retMsg = producer.request(msg, ttl); + long cost = System.currentTimeMillis() - begin; + System.err.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); } catch (Exception e) { log.warn("", e); } From 53a6955e5b3945b2783fdc58900fca116410f505 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Thu, 12 Sep 2019 10:40:21 +0800 Subject: [PATCH 07/73] [RIP-16]change the way creating reply message --- .../java/org/apache/rocketmq/client/utils/MessageUtil.java | 3 ++- .../org/apache/rocketmq/example/rpc/ResponseConsumer.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index 81da669ff8..e60c0f665f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -24,13 +24,14 @@ import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; public class MessageUtil { - public static Message createReplyMessage(final Message requestMessage) throws MQClientException { + public static Message createReplyMessage(final Message requestMessage, final byte[] body) throws MQClientException { if (requestMessage != null) { Message replyMessage = new Message(); String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); String requestUniqId = requestMessage.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); + replyMessage.setBody(body); if (cluster != null) { String replyTopic = MixAll.getReplyTopic(cluster); replyMessage.setTopic(replyTopic); diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index ff3652fe3a..3580f87fb5 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -56,8 +56,8 @@ public class ResponseConsumer { String replyTo = msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); //You must use MessageUtil to creage reply message, otherwise reply message maybe wrong. - Message replyMessage = MessageUtil.createReplyMessage(msg); - replyMessage.setBody("reply message contents.".getBytes()); + byte[] replyContent = "reply message contents.".getBytes(); + Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent); //maybe you should create a producer to send reply message. SendResult sendResult = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(replyMessage, 3000); From ec1f9266d2bcd95c9ac41e049d15fec20c9d4afa Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 18 Sep 2019 10:48:15 +0800 Subject: [PATCH 08/73] optimize request/response examples --- .../apache/rocketmq/example/rpc/AsyncRequestProducer.java | 5 ++--- .../org/apache/rocketmq/example/rpc/RequestProducer.java | 4 +--- .../org/apache/rocketmq/example/rpc/ResponseConsumer.java | 4 +--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java index d2475c165b..072291d5c2 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/AsyncRequestProducer.java @@ -29,13 +29,11 @@ public class AsyncRequestProducer { private static final InternalLogger log = ClientLogger.getLog(); public static void main(String[] args) throws MQClientException, InterruptedException { - String producerGroup = "ProducerGroup-Name"; - String namesrvAddr = "10.255.2.37:9876;10.255.2.37:9875"; + String producerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; long ttl = 3000; DefaultMQProducer producer = new DefaultMQProducer(producerGroup); - producer.setNamesrvAddr(namesrvAddr); producer.start(); try { @@ -59,6 +57,7 @@ public class AsyncRequestProducer { } catch (Exception e) { log.warn("", e); } + /* shutdown after your request callback is finished */ // producer.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java index a83f1db87f..b038151a88 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java @@ -28,13 +28,11 @@ public class RequestProducer { private static final InternalLogger log = ClientLogger.getLog(); public static void main(String[] args) throws MQClientException, InterruptedException { - String producerGroup = "ProducerGroup-Name"; - String namesrvAddr = "10.255.2.37:9876;10.255.2.37:9875"; + String producerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; long ttl = 3000; DefaultMQProducer producer = new DefaultMQProducer(producerGroup); - producer.setNamesrvAddr(namesrvAddr); producer.start(); try { diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index 3580f87fb5..131fc9dfcc 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -38,12 +38,10 @@ public class ResponseConsumer { private static final InternalLogger log = ClientLogger.getLog(); public static void main(String[] args) throws InterruptedException, MQClientException { - String consumerGroup = "ConsumeGroup-Name"; - String namesrvAddr = "10.255.2.37:9873"; + String consumerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup); - consumer.setNamesrvAddr(namesrvAddr); consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); consumer.registerMessageListener(new MessageListenerConcurrently() { From 5e1732a9c4063f4b299684e172409f74256bd5c2 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 18 Sep 2019 15:00:05 +0800 Subject: [PATCH 09/73] add unittest of request/response pattern --- .../processor/ReplyMessageProcessorTest.java | 141 ++++++++++++++++++ .../client/impl/MQClientAPIImplTest.java | 36 ++++- .../producer/DefaultMQProducerTest.java | 111 +++++++++++++- .../producer/RequestResponseFutureTest.java | 45 ++++++ .../client/utils/MessageUtilsTest.java | 62 ++++++++ 5 files changed, 388 insertions(+), 7 deletions(-) create mode 100644 broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java create mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java create mode 100644 client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java new file mode 100644 index 0000000000..f48594ebbe --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.net.Broker2Client; +import org.apache.rocketmq.broker.transaction.TransactionalMessageService; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.NettyServerConfig; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.apache.rocketmq.store.AppendMessageResult; +import org.apache.rocketmq.store.AppendMessageStatus; +import org.apache.rocketmq.store.MessageExtBrokerInner; +import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ReplyMessageProcessorTest { + private ReplyMessageProcessor replyMessageProcessor; + @Spy + private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), new NettyClientConfig(), new MessageStoreConfig()); + @Mock + private ChannelHandlerContext handlerContext; + @Mock + private MessageStore messageStore; + @Mock + private Channel channel; + @Mock + private TransactionalMessageService transactionMsgService; + private String topic = "FooBar"; + private String group = "FooBarGroup"; + private ClientChannelInfo clientInfo; + @Mock + private Broker2Client broker2Client; + + @Before + public void init() throws IllegalAccessException, NoSuchFieldException { + clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); + brokerController.setMessageStore(messageStore); + Field field = BrokerController.class.getDeclaredField("broker2Client"); + field.setAccessible(true); + field.set(brokerController, broker2Client); + when(messageStore.now()).thenReturn(System.currentTimeMillis()); + Channel mockChannel = mock(Channel.class); + when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024)); + when(handlerContext.channel()).thenReturn(mockChannel); + when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); + replyMessageProcessor = new ReplyMessageProcessor(brokerController); + } + + @Test + public void testProcessRequest_Success() throws RemotingCommandException, InterruptedException, RemotingTimeoutException, RemotingSendRequestException { + when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult(PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + brokerController.getProducerManager().registerProducer(group, clientInfo); + final RemotingCommand request = createSendMessageRequestHeaderCommand(RequestCode.SEND_REPLY_MESSAGE); + when(brokerController.getBroker2Client().callClient(any(Channel.class), any(RemotingCommand.class))).thenReturn(createResponse(ResponseCode.SUCCESS, request)); + RemotingCommand responseToReturn = replyMessageProcessor.processRequest(handlerContext, request); + assertThat(responseToReturn.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(responseToReturn.getOpaque()).isEqualTo(request.getOpaque()); + } + + private RemotingCommand createSendMessageRequestHeaderCommand(int requestCode) { + SendMessageRequestHeader requestHeader = createSendMessageRequestHeader(); + RemotingCommand request = RemotingCommand.createRequestCommand(requestCode, requestHeader); + request.setBody(new byte[] {'a'}); + request.makeCustomHeaderToNet(); + return request; + } + + private SendMessageRequestHeader createSendMessageRequestHeader() { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(group); + requestHeader.setTopic(topic); + requestHeader.setDefaultTopic(MixAll.AUTO_CREATE_TOPIC_KEY_TOPIC); + requestHeader.setDefaultTopicQueueNums(3); + requestHeader.setQueueId(1); + requestHeader.setSysFlag(0); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(124); + requestHeader.setReconsumeTimes(0); + Map map = new HashMap(); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "127.0.0.1"); + requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); + return requestHeader; + } + + private RemotingCommand createResponse(int code, RemotingCommand request) { + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setCode(code); + response.setOpaque(request.getOpaque()); + return response; + } +} \ No newline at end of file diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index 84af632355..3f00d9e403 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -42,6 +42,7 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Matchers; import org.mockito.Mock; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; @@ -164,7 +165,7 @@ public class MQClientAPIImplTest { public Object answer(InvocationOnMock mock) throws Throwable { InvokeCallback callback = mock.getArgument(3); RemotingCommand request = mock.getArgument(1); - ResponseFuture responseFuture = new ResponseFuture(null,request.getOpaque(), 3 * 1000, null, null); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); responseFuture.setResponseCommand(createSuccessResponse(request)); callback.operationComplete(responseFuture); return null; @@ -289,6 +290,7 @@ public class MQClientAPIImplTest { assertThat(ex.getErrorMessage()).isEqualTo("corresponding to accessConfig has been deleted failed"); } } + @Test public void testResumeCheckHalfMessage_WithException() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { doAnswer(new Answer() { @@ -322,6 +324,38 @@ public class MQClientAPIImplTest { assertThat(result).isEqualTo(true); } + @Test + public void testSendMessageTypeofReply() throws Exception { + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + InvokeCallback callback = mock.getArgument(3); + RemotingCommand request = mock.getArgument(1); + ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); + responseFuture.setResponseCommand(createSuccessResponse(request)); + callback.operationComplete(responseFuture); + return null; + } + }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); + SendMessageContext sendMessageContext = new SendMessageContext(); + sendMessageContext.setProducer(new DefaultMQProducerImpl(new DefaultMQProducer())); + msg.getProperties().put("MSG_TYPE", "reply"); + mqClientAPI.sendMessage(brokerAddr, brokerName, msg, new SendMessageRequestHeader(), 3 * 1000, CommunicationMode.ASYNC, + new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); + assertThat(sendResult.getQueueOffset()).isEqualTo(123L); + assertThat(sendResult.getMessageQueue().getQueueId()).isEqualTo(1); + } + + @Override + public void onException(Throwable e) { + } + }, null, null, 0, sendMessageContext, defaultMQProducerImpl); + } + private RemotingCommand createResumeSuccessResponse(RemotingCommand request) { RemotingCommand response = RemotingCommand.createResponseCommand(null); response.setCode(ResponseCode.SUCCESS); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 9540755fe3..6f2f959632 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -30,6 +32,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.RequestTimeoutException; import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.hook.SendMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; @@ -45,6 +48,7 @@ import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.QueueData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; +import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.netty.NettyRemotingClient; import org.junit.After; import org.junit.Before; @@ -184,6 +188,7 @@ public class DefaultMQProducerTest { }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); } + @Test public void testSendMessageAsync() throws RemotingException, MQClientException, InterruptedException { final AtomicInteger cc = new AtomicInteger(0); @@ -211,12 +216,12 @@ public class DefaultMQProducerTest { Message message = new Message(); message.setTopic("test"); message.setBody("hello world".getBytes()); - producer.send(new Message(),sendCallback); - producer.send(message,sendCallback,1000); - producer.send(message,new MessageQueue(),sendCallback); - producer.send(new Message(),new MessageQueue(),sendCallback,1000); - producer.send(new Message(),messageQueueSelector,null,sendCallback); - producer.send(message,messageQueueSelector,null,sendCallback,1000); + producer.send(new Message(), sendCallback); + producer.send(message, sendCallback, 1000); + producer.send(message, new MessageQueue(), sendCallback); + producer.send(new Message(), new MessageQueue(), sendCallback, 1000); + producer.send(new Message(), messageQueueSelector, null, sendCallback); + producer.send(message, messageQueueSelector, null, sendCallback, 1000); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(6); @@ -311,6 +316,100 @@ public class DefaultMQProducerTest { assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); } + @Test + public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + new Thread(new Runnable() { + @Override public void run() { + boolean flag = true; + ConcurrentHashMap responseMap = null; + while (flag) { + responseMap = RequestFutureTable.getRequestFutureTable(); + if (responseMap != null) { + flag = false; + } + } + assertThat(responseMap).isNotNull(); + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.putResponseMessage(message); + } + } + }).start(); + Message result = producer.request(message, 3 * 1000L); + assertThat(result.getTopic()).isEqualTo("FooBar"); + assertThat(result.getBody()).isEqualTo(new byte[] {'a'}); + } + + @Test(expected = RequestTimeoutException.class) + public void testRequestMessage_RequestTimeoutException() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + Message result = producer.request(message, 3 * 1000L); + } + + @Test + public void testAsyncRequest_OnSuccess() throws Exception { + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final CountDownLatch countDownLatch = new CountDownLatch(1); + RequestCallback requestCallback = new RequestCallback() { + @Override public void onSuccess(Message message) { + assertThat(message.getTopic()).isEqualTo("FooBar"); + assertThat(message.getBody()).isEqualTo(new byte[] {'a'}); + assertThat(message.getFlag()).isEqualTo(1); + countDownLatch.countDown(); + } + + @Override public void onException(Throwable e) { + } + }; + producer.request(message, requestCallback, 3 * 1000L); + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); + assertThat(responseMap).isNotNull(); + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.setSendReqeustOk(true); + message.setFlag(1); + future.getRequestCallback().onSuccess(message); + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + } + + @Test + public void testAsyncRequest_OnException() throws Exception { + final AtomicInteger cc = new AtomicInteger(0); + final CountDownLatch countDownLatch = new CountDownLatch(1); + RequestCallback requestCallback = new RequestCallback() { + @Override public void onSuccess(Message message) { + + } + + @Override public void onException(Throwable e) { + cc.incrementAndGet(); + countDownLatch.countDown(); + } + }; + MessageQueueSelector messageQueueSelector = new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + return null; + } + }; + + try { + producer.request(message, requestCallback, 3 * 1000L); + failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); + } catch (RemotingSendRequestException e) { + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); + assertThat(responseMap).isNotNull(); + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.getRequestCallback().onException(new Throwable()); + } + } + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); + } + public static TopicRouteData createTopicRoute() { TopicRouteData topicRouteData = new TopicRouteData(); diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java new file mode 100644 index 0000000000..90e4623e9b --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/RequestResponseFutureTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.producer; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.common.message.Message; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class RequestResponseFutureTest { + + @Test + public void testExecuteRequestCallback() throws Exception { + final AtomicInteger cc = new AtomicInteger(0); + RequestResponseFuture future = new RequestResponseFuture(UUID.randomUUID().toString(), 3 * 1000L, new RequestCallback() { + @Override public void onSuccess(Message message) { + cc.incrementAndGet(); + } + + @Override public void onException(Throwable e) { + } + }); + future.setSendReqeustOk(true); + future.executeRequestCallback(); + assertThat(cc.get()).isEqualTo(1); + } + +} diff --git a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java new file mode 100644 index 0000000000..a722fa6cbb --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.utils; + +import java.util.HashMap; +import java.util.Map; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; + +public class MessageUtilsTest { + + @Test + public void testCreateReplyMessage() throws MQClientException { + Message msg = MessageUtil.createReplyMessage(createReplyMessage("clusterName"), new byte[] {'a'}); + assertThat(msg.getTopic()).isEqualTo("clusterName" + "_" + MixAll.REPLY_TOPIC_POSTFIX); + assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO)).isEqualTo("127.0.0.1"); + assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_TTL)).isEqualTo("3000"); + } + + @Test + public void testCreateReplyMessage_Exception() throws MQClientException { + try { + Message msg = MessageUtil.createReplyMessage(createReplyMessage(null), new byte[] {'a'}); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("create reply message fail."); + } + } + + private Message createReplyMessage(String clusterName) { + Message requestMessage = new Message(); + Map map = new HashMap(); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "127.0.0.1"); + map.put(MessageConst.PROPERTY_CLUSTER, clusterName); + map.put(MessageConst.PROPERTY_MESSAGE_TTL, "3000"); + MessageAccessor.setProperties(requestMessage, map); + return requestMessage; + } + +} From 21511da384dea69b734ca1d5a3bd5c84267ecea2 Mon Sep 17 00:00:00 2001 From: rongtongjin <794220751@qq.com> Date: Fri, 20 Sep 2019 13:43:53 +0800 Subject: [PATCH 10/73] fix(broker):fix the issue that master node route info is lost when some node join the group in dledger mode Since brokerId defaults to 0, it will register with nameserver before brokerId is changed when broker restart, causing the route info of master to be overwritten. Set brokerId defaults to -1 in dledger mode and do not register wtih nameserver until brokerId is changed. Close #1464 --- .../java/org/apache/rocketmq/broker/BrokerController.java | 5 +---- .../main/java/org/apache/rocketmq/broker/BrokerStartup.java | 4 ++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index a885cd0898..9bc154fd92 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -857,12 +857,9 @@ public class BrokerController { if (!messageStoreConfig.isEnableDLegerCommitLog()) { startProcessorByHa(messageStoreConfig.getBrokerRole()); handleSlaveSynchronize(messageStoreConfig.getBrokerRole()); + this.registerBrokerAll(true, false, true); } - - - this.registerBrokerAll(true, false, true); - this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java index 4b986c0d24..960b848461 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerStartup.java @@ -178,6 +178,10 @@ public class BrokerStartup { break; } + if (messageStoreConfig.isEnableDLegerCommitLog()) { + brokerConfig.setBrokerId(-1); + } + messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1); LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); JoranConfigurator configurator = new JoranConfigurator(); From 4c4bf104b5f898a0263f2effd22ac9e652cf36fd Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Sat, 21 Sep 2019 23:18:53 +0800 Subject: [PATCH 11/73] add unittest of request/response pattern --- .../broker/client/ProducerManagerTest.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index c47a0cb7c6..6271138313 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -50,7 +50,7 @@ public class ProducerManagerTest { public void scanNotActiveChannel() throws Exception { producerManager.registerProducer(group, clientInfo); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); - + assertThat(producerManager.findChannel("clientId")).isNotNull(); Field field = ProducerManager.class.getDeclaredField("CHANNEL_EXPIRED_TIMEOUT"); field.setAccessible(true); long CHANNEL_EXPIRED_TIMEOUT = field.getLong(producerManager); @@ -58,22 +58,28 @@ public class ProducerManagerTest { when(channel.close()).thenReturn(mock(ChannelFuture.class)); producerManager.scanNotActiveChannel(); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.findChannel("clientId")).isNull(); } @Test public void doChannelCloseEvent() throws Exception { producerManager.registerProducer(group, clientInfo); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); + assertThat(producerManager.findChannel("clientId")).isNotNull(); producerManager.doChannelCloseEvent("127.0.0.1", channel); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.findChannel("clientId")).isNull(); } @Test public void testRegisterProducer() throws Exception { producerManager.registerProducer(group, clientInfo); HashMap channelMap = producerManager.getGroupChannelTable().get(group); + Channel channel1 = producerManager.findChannel("clientId"); assertThat(channelMap).isNotNull(); + assertThat(channel1).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientInfo); + assertThat(channel1).isEqualTo(channel); } @Test @@ -82,10 +88,15 @@ public class ProducerManagerTest { HashMap channelMap = producerManager.getGroupChannelTable().get(group); assertThat(channelMap).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientInfo); - + Channel channel1 = producerManager.findChannel("clientId"); + assertThat(channel1).isNotNull(); + assertThat(channel1).isEqualTo(channel); producerManager.unregisterProducer(group, clientInfo); channelMap = producerManager.getGroupChannelTable().get(group); + channel1 = producerManager.findChannel("clientId"); assertThat(channelMap).isNull(); + assertThat(channel1).isNull(); + } } \ No newline at end of file From 28936fa12a21b13e9d5813385c6d59727d27b839 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Mon, 23 Sep 2019 20:07:41 +0800 Subject: [PATCH 12/73] add reply interface to consumer --- .../consumer/DefaultMQPullConsumer.java | 49 ++++++++ .../consumer/DefaultMQPushConsumer.java | 49 ++++++++ .../client/impl/ClientRemotingProcessor.java | 7 +- .../consumer/DefaultMQPullConsumerImpl.java | 24 ++++ .../consumer/DefaultMQPushConsumerImpl.java | 29 ++++- .../client/impl/consumer/MQConsumerInner.java | 15 +++ .../impl/producer/DefaultMQProducerImpl.java | 50 ++++---- .../client/producer/DefaultMQProducer.java | 118 +++++++++++------- .../producer/RequestResponseFuture.java | 20 +-- .../rocketmq/client/utils/MessageUtil.java | 4 +- .../rocketmq/example/rpc/RequestProducer.java | 8 +- .../example/rpc/ResponseConsumer.java | 24 +--- 12 files changed, 275 insertions(+), 122 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index f3b6caaa71..17e60080e0 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -25,7 +25,10 @@ import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -444,4 +447,50 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume public void setMaxReconsumeTimes(final int maxReconsumeTimes) { this.maxReconsumeTimes = maxReconsumeTimes; } + + /** + * send a reply message to the producer of the original request message + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param timeoutMillis + * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + return this.defaultMQPullConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message asynchronously + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param replyCallback callback to execute on replying completion. + * @param timeoutMillis + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultMQPullConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message oneway + * @param requestMsg original request message + * @param replyContent contents of reply message + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultMQPullConsumerImpl.replyOneway(requestMsg, replyContent); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 339f799f9a..8aa12a49a2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -30,12 +30,15 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -889,4 +892,50 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume public TraceDispatcher getTraceDispatcher() { return traceDispatcher; } + + /** + * send a reply message to the producer of the original request message + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param timeoutMillis + * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + return this.defaultMQPushConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message asynchronously + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param replyCallback callback to execute on replying completion. + * @param timeoutMillis + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultMQPushConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message oneway + * @param requestMsg original request message + * @param replyContent contents of reply message + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultMQPushConsumerImpl.replyOneway(requestMsg, replyContent); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index e9072df3a6..3bea2d8e5d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -268,13 +268,14 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { log.debug("receive reply message :{}", msg); processReplyMessage(msg); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); } catch (Exception e) { - log.warn("unknown err when receiveRRReplyMsg", e); + log.warn("unknown err when receiveReplyMsg", e); response.setCode(ResponseCode.SYSTEM_ERROR); response.setRemark("process reply message fail"); } - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); return response; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 598dc94cc2..9975cb835e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -43,6 +43,9 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; @@ -796,4 +799,25 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { public RebalanceImpl getRebalanceImpl() { return rebalanceImpl; } + + @Override + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); + } + + @Override + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); + } + + @Override + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 48fe41ada6..8615746ee2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -27,7 +27,6 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentMap; - import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; @@ -51,20 +50,22 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.stat.ConsumerStatsManager; +import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.protocol.NamespaceUtil; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.ProcessQueueInfo; @@ -75,6 +76,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -1168,4 +1170,25 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { this.consumeMessageService = consumeMessageService; } + + @Override + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); + } + + @Override + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); + } + + @Override + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java index c2e8a1dfc4..010d1db345 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java @@ -17,12 +17,18 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Set; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.exception.RemotingException; /** * Consumer inner interface @@ -49,4 +55,13 @@ public interface MQConsumerInner { boolean isUnitMode(); ConsumerRunningInfo consumerRunningInfo(); + + SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; + + void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; + + void replyOneway(final Message requestMsg, + final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 6e15ed1a09..180ce1974e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -1334,6 +1334,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public Message request(Message msg, long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1341,6 +1342,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1353,11 +1355,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.putResponseMessage(null); requestResponseFuture.setCause(e); } - }, timeout); + }, timeout - cost); - Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); if (responseMessage == null) { - if (requestResponseFuture.isSendReqeustOk()) { + if (requestResponseFuture.isSendRequestOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { @@ -1371,6 +1373,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void request(Message msg, final RequestCallback requestCallback, long timeout) throws RemotingException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1378,6 +1381,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1389,7 +1393,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.setCause(e); requestFail(requestUniqId); } - }, timeout); + }, timeout - cost); } catch (Exception ex) { log.warn("send request message to <{}> failed.", msg.getTopic(), ex); throw new RemotingSendRequestException(msg.getTopic(), ex); @@ -1399,6 +1403,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1406,6 +1411,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1418,11 +1424,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.putResponseMessage(null); requestResponseFuture.setCause(e); } - }, timeout); + }, timeout - cost); - Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); if (responseMessage == null) { - if (requestResponseFuture.isSendReqeustOk()) { + if (requestResponseFuture.isSendRequestOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { @@ -1437,6 +1443,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public void request(final Message msg, final MessageQueueSelector selector, final Object arg, final RequestCallback requestCallback, final long timeout) throws RemotingException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1444,6 +1451,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1455,7 +1463,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.setCause(e); requestFail(requestUniqId); } - }, timeout); + }, timeout - cost); } catch (Exception ex) { log.warn("send request message to <{}> failed.", msg.getTopic(), ex); throw new RemotingSendRequestException(msg.getTopic(), ex); @@ -1464,6 +1472,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public Message request(final Message msg, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1471,6 +1480,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1483,11 +1493,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.putResponseMessage(null); requestResponseFuture.setCause(e); } - }, null, timeout); + }, null, timeout - cost); - Message responseMessage = requestResponseFuture.waitResponseMessage(timeout); + Message responseMessage = requestResponseFuture.waitResponseMessage(timeout - cost); if (responseMessage == null) { - if (requestResponseFuture.isSendReqeustOk()) { + if (requestResponseFuture.isSendRequestOk()) { throw new RequestTimeoutException(ClientErrorCode.REQUEST_TIMEOUT_EXCEPTION, "send request message to <" + msg.getTopic() + "> OK, but wait reply message timeout, " + timeout + " ms."); } else { @@ -1502,6 +1512,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) throws RemotingException { + long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); @@ -1509,6 +1520,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + long cost = System.currentTimeMillis() - beginTimestamp; this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -1520,7 +1532,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { requestResponseFuture.setCause(e); requestFail(requestUniqId); } - }, null, timeout); + }, null, timeout - cost); } catch (Exception ex) { log.warn("send request message to <{}> failed.", msg.getTopic(), ex); throw new RemotingSendRequestException(msg.getTopic(), ex); @@ -1559,20 +1571,6 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } - private SendResult reply(final Message msg, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeoutMillis); - } - - private SendResult reply(final Message msg, final SendCallback sendCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - return this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeoutMillis); - } - - public void replyOneway(final Message msg) throws RemotingException, MQClientException, InterruptedException { - this.sendOneway(msg); - } - public ConcurrentMap getTopicPublishInfoTable() { return topicPublishInfoTable; } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index bb017ef212..fdb489eebd 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -566,13 +566,19 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * @param msg - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws MQBrokerException - * @throws InterruptedException + * Send request message in synchronous mode. This method returns only when the consumer consume the request message and reply a message.

+ * + * Warn: this method has internal retry-mechanism, that is, internal implementation will retry + * {@link #retryTimesWhenSendFailed} times before claiming failure. As a result, multiple messages may potentially + * delivered to broker(s). It's up to the application developers to resolve potential duplication issue. + * + * @param msg request message to send + * @param timeout request timeout + * @return reply message + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. */ @Override public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, @@ -582,85 +588,101 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { } /** - * @param msg - * @param requestCallback - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException + * Request asynchronously.

+ * This method returns immediately. On receiving reply message, requestCallback will be executed.

+ * + * Similar to {@link #request(Message, long)}, internal implementation would potentially retry up to {@link + * #retryTimesWhenSendAsyncFailed} times before claiming sending failure, which may yield message duplication and + * application developers are the one to resolve this potential issue. + * + * @param msg request message to send + * @param requestCallback callback to execute on request completion. + * @param timeout request timeout + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. */ @Override public void request(final Message msg, final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException, MQBrokerException, RequestTimeoutException { + msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, requestCallback, timeout); } /** - * @param msg - * @param selector - * @param arg - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws MQBrokerException - * @throws InterruptedException + * Same to {@link #request(Message, long)} with message queue selector specified. + * + * @param msg request message to send + * @param selector message queue selector, through which we get target message queue to deliver message to. + * @param arg argument to work along with message queue selector. + * @param timeout timeout of request. + * @return reply message + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. */ @Override public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + msg.setTopic(withNamespace(msg.getTopic())); return this.defaultMQProducerImpl.request(msg, selector, arg, timeout); } /** - * @param msg - * @param selector - * @param arg - * @param requestCallback - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException + * Same to {@link #request(Message, RequestCallback, long)} with target message selector specified. + * + * @param msg requst message to send + * @param selector message queue selector, through which we get target message queue to deliver message to. + * @param arg argument to work along with message queue selector. + * @param requestCallback callback to execute on request completion. + * @param timeout timeout of request. + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. */ @Override public void request(final Message msg, final MessageQueueSelector selector, final Object arg, final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, InterruptedException { + msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout); } /** - * @param msg - * @param mq - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws MQBrokerException - * @throws InterruptedException + * Same to {@link #request(Message, long)} with target message queue specified in addition. + * + * @param msg request message to send + * @param mq target message queue. + * @param timeout request timeout + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws MQBrokerException if there is any broker error. + * @throws InterruptedException if the thread is interrupted. */ @Override public Message request(final Message msg, final MessageQueue mq, final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { + msg.setTopic(withNamespace(msg.getTopic())); return this.defaultMQProducerImpl.request(msg, mq, timeout); } /** - * @param msg - * @param mq - * @param requestCallback - * @param timeout - * @return - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException + * Same to {@link #request(Message, RequestCallback, long)} with target message queue specified. + * + * @param msg request message to send + * @param mq target message queue. + * @param requestCallback callback to execute on request completion. + * @param timeout timeout of request. + * @throws MQClientException if there is any client error. + * @throws RemotingException if there is any network-tier error. + * @throws InterruptedException if the thread is interrupted. */ @Override public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) throws MQClientException, RemotingException, InterruptedException { + msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java index d65b479434..c476ba30f1 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -19,7 +19,6 @@ package org.apache.rocketmq.client.producer; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.common.message.Message; public class RequestResponseFuture { @@ -29,9 +28,8 @@ public class RequestResponseFuture { private final Message requestMsg = null; private long timeoutMillis; private CountDownLatch countDownLatch = new CountDownLatch(1); - private AtomicBoolean ececuteCallbackOnlyOnce = new AtomicBoolean(false); private volatile Message responseMsg = null; - private volatile boolean sendReqeustOk = true; + private volatile boolean sendRequestOk = true; private volatile Throwable cause = null; public RequestResponseFuture(String requestUniqId, long timeoutMillis, RequestCallback requestCallback) { @@ -42,7 +40,7 @@ public class RequestResponseFuture { public void executeRequestCallback() { if (requestCallback != null) { - if (sendReqeustOk && cause == null) { + if (sendRequestOk && cause == null) { requestCallback.onSuccess(responseMsg); } else { requestCallback.onException(cause); @@ -93,14 +91,6 @@ public class RequestResponseFuture { this.countDownLatch = countDownLatch; } - public AtomicBoolean getEcecuteCallbackOnlyOnce() { - return ececuteCallbackOnlyOnce; - } - - public void setEcecuteCallbackOnlyOnce(AtomicBoolean ececuteCallbackOnlyOnce) { - this.ececuteCallbackOnlyOnce = ececuteCallbackOnlyOnce; - } - public Message getResponseMsg() { return responseMsg; } @@ -109,12 +99,12 @@ public class RequestResponseFuture { this.responseMsg = responseMsg; } - public boolean isSendReqeustOk() { - return sendReqeustOk; + public boolean isSendRequestOk() { + return sendRequestOk; } public void setSendReqeustOk(boolean sendReqeustOk) { - this.sendReqeustOk = sendReqeustOk; + this.sendRequestOk = sendReqeustOk; } public Message getRequestMsg() { diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index e60c0f665f..1e975a9a98 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -41,8 +41,10 @@ public class MessageUtil { MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); return replyMessage; + } else { + throw new MQClientException(-1, "create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); } } - throw new MQClientException(-1, "create reply message fail."); + throw new MQClientException(-1, "create reply message fail, requestMessage cannot be null."); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java index b038151a88..b34908b84f 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/RequestProducer.java @@ -18,15 +18,11 @@ package org.apache.rocketmq.example.rpc; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; public class RequestProducer { - private static final InternalLogger log = ClientLogger.getLog(); - public static void main(String[] args) throws MQClientException, InterruptedException { String producerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; @@ -43,9 +39,9 @@ public class RequestProducer { long begin = System.currentTimeMillis(); Message retMsg = producer.request(msg, ttl); long cost = System.currentTimeMillis() - begin; - System.err.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); + System.out.printf("request to <%s> cost: %d replyMessage: %s %n", topic, cost, retMsg); } catch (Exception e) { - log.warn("", e); + e.printStackTrace(); } producer.shutdown(); } diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index 131fc9dfcc..a130722a71 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -24,19 +24,13 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.exception.RemotingException; public class ResponseConsumer { - private static final InternalLogger log = ClientLogger.getLog(); - public static void main(String[] args) throws InterruptedException, MQClientException { String consumerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; @@ -50,23 +44,13 @@ public class ResponseConsumer { System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); for (MessageExt msg : msgs) { try { - log.info("handle message: {} body={}", msg, new String(msg.getBody())); + System.out.printf("handle message: %s", msg.toString()); String replyTo = msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); - //You must use MessageUtil to creage reply message, otherwise reply message maybe wrong. byte[] replyContent = "reply message contents.".getBytes(); - Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent); - - //maybe you should create a producer to send reply message. - SendResult sendResult = consumer.getDefaultMQPushConsumerImpl().getmQClientFactory().getDefaultMQProducer().send(replyMessage, 3000); - System.out.printf("reply msg %s to %s , %s %n", replyMessage.toString(), replyTo, sendResult.toString()); - } catch (MQClientException e) { - e.printStackTrace(); - } catch (RemotingException e) { - e.printStackTrace(); - } catch (MQBrokerException e) { - e.printStackTrace(); - } catch (InterruptedException e) { + SendResult replyResult = consumer.reply(msg, replyContent, 3000); + System.out.printf("reply to %s , %s %n", replyTo, replyResult.toString()); + } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { e.printStackTrace(); } } From 8c372b4596d64f90e41e38534820baf655b3f1a1 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Mon, 23 Sep 2019 21:30:56 +0800 Subject: [PATCH 13/73] optimize ReplyMessageProcessor --- .../processor/ReplyMessageProcessor.java | 146 ++++++++++-------- 1 file changed, 85 insertions(+), 61 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index 23e1e9ac8b..265c5c122f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -23,6 +23,7 @@ import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; @@ -34,6 +35,8 @@ import org.apache.rocketmq.common.protocol.header.ReplyMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; @@ -43,6 +46,7 @@ import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.stats.BrokerStatsManager; public class ReplyMessageProcessor extends AbstractSendMessageProcessor implements NettyRequestProcessor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); public ReplyMessageProcessor(final BrokerController brokerController) { super(brokerController); @@ -136,22 +140,20 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen msgInner.setStoreHost(this.getStoreHost()); msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader.getReconsumeTimes()); - boolean pushOk = this.pushReplyMessage(ctx, requestHeader, msgInner, response); + PushReplyResult pushReplyResult = this.pushReplyMessage(ctx, requestHeader, msgInner); + this.handlePushReplyResult(pushReplyResult, response, responseHeader, queueIdInt); - if (pushOk && this.brokerController.getBrokerConfig().isStoreReplyMessageEnable()) { + if (this.brokerController.getBrokerConfig().isStoreReplyMessageEnable()) { PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); - this.handlePutMessageResult(putMessageResult, response, request, msgInner, responseHeader, sendMessageContext, ctx, queueIdInt); - } else { - responseHeader.setMsgId("0"); - responseHeader.setQueueId(0); - responseHeader.setQueueOffset(0L); + this.handlePutMessageResult(putMessageResult, request, msgInner, responseHeader, sendMessageContext, queueIdInt); } return response; } - private boolean pushReplyMessage(final ChannelHandlerContext ctx, final SendMessageRequestHeader requestHeader, - final Message msg, final RemotingCommand response) { + private PushReplyResult pushReplyMessage(final ChannelHandlerContext ctx, + final SendMessageRequestHeader requestHeader, + final Message msg) { ReplyMessageRequestHeader replyMessageRequestHeader = new ReplyMessageRequestHeader(); replyMessageRequestHeader.setBornHost(ctx.channel().remoteAddress().toString()); replyMessageRequestHeader.setStoreHost(this.getStoreHost().toString()); @@ -172,7 +174,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen request.setBody(msg.getBody()); String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO); - boolean pushOk = false; + PushReplyResult pushReplyResult = new PushReplyResult(false); if (senderId != null) { Channel channel = this.brokerController.getProducerManager().findChannel(senderId); @@ -185,106 +187,102 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen assert pushResponse != null; switch (pushResponse.getCode()) { case ResponseCode.SUCCESS: { - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); - pushOk = true; + pushReplyResult.setPushOk(true); break; } default: { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("push reply message to requester fail"); - log.warn("push reply message to <{}> return fail, remark: {}", senderId, response.getRemark()); + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message to " + senderId + "fail."); + log.warn("push reply message to <{}> return fail, response remark: {}", senderId, pushResponse.getRemark()); } } - } catch (InterruptedException e) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("push reply message to requester fail"); - log.warn("push reply message to <{}> fail. {}", senderId, channel, e); - } catch (RemotingException e) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("push reply message to requester fail"); + } catch (RemotingException | InterruptedException e) { + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message to " + senderId + "fail."); log.warn("push reply message to <{}> fail. {}", senderId, channel, e); } } else { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("channel of <" + senderId + "> not found"); - log.warn("push reply message fial, channel of <{}> not found.", senderId); + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("push reply message fail, channel of <" + senderId + "> not found."); + log.warn(pushReplyResult.getRemark()); } - return pushOk; + } else { + log.warn("REPLY_TO is null, can not reply message"); + pushReplyResult.setPushOk(false); + pushReplyResult.setRemark("reply message properties[" + MessageConst.PROPERTY_MESSAGE_REPLY_TO + "] is null"); } - log.warn("REPLY_TO is null, can not reply message"); - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("REPLY_TO is null"); - return pushOk; + return pushReplyResult; } - private void handlePutMessageResult(PutMessageResult putMessageResult, final RemotingCommand response, + private void handlePushReplyResult(PushReplyResult pushReplyResult, final RemotingCommand response, + final SendMessageResponseHeader responseHeader, int queueIdInt) { + + if (!pushReplyResult.isPushOk()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(pushReplyResult.getRemark()); + } else { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + //set to zore to avoid client decoding exception + responseHeader.setMsgId("0"); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(0L); + } + } + + private void handlePutMessageResult(PutMessageResult putMessageResult, final RemotingCommand request, final MessageExt msg, final SendMessageResponseHeader responseHeader, SendMessageContext sendMessageContext, - ChannelHandlerContext ctx, int queueIdInt) { if (putMessageResult == null) { - response.setRemark("push reply to requester success, but store putMessage return null"); + log.warn("process reply message, store putMessage return null"); return; } - boolean sendOK = false; + boolean putOk = false; switch (putMessageResult.getPutMessageStatus()) { // Success case PUT_OK: - sendOK = true; - response.setCode(ResponseCode.SUCCESS); - break; case FLUSH_DISK_TIMEOUT: - response.setCode(ResponseCode.FLUSH_DISK_TIMEOUT); - sendOK = true; - break; case FLUSH_SLAVE_TIMEOUT: - response.setCode(ResponseCode.FLUSH_SLAVE_TIMEOUT); - sendOK = true; - break; case SLAVE_NOT_AVAILABLE: - response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); - sendOK = true; + putOk = true; break; // Failed case CREATE_MAPEDFILE_FAILED: -// response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("create mapped file failed, server is busy or broken."); + log.info("create mapped file failed, server is busy or broken."); break; case MESSAGE_ILLEGAL: + log.info( + "the message is illegal, maybe msg properties length limit 32k."); + break; case PROPERTIES_SIZE_EXCEEDED: -// response.setCode(ResponseCode.MESSAGE_ILLEGAL); - response.setRemark( - "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k, msg properties length limit 32k."); + log.info( + "the message is illegal, maybe msg body or properties length not matched. msg body length limit 128k."); break; case SERVICE_NOT_AVAILABLE: -// response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); - response.setRemark( + log.info( "service not available now, maybe disk full, maybe your broker machine memory too small."); break; case OS_PAGECACHE_BUSY: -// response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); + log.info("[PC_SYNCHRONIZED]broker busy, start flow control for a while"); break; case UNKNOWN_ERROR: -// response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR"); + log.info("UNKNOWN_ERROR"); break; default: -// response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark("UNKNOWN_ERROR DEFAULT"); + log.info("UNKNOWN_ERROR DEFAULT"); break; } String owner = request.getExtFields().get(BrokerStatsManager.COMMERCIAL_OWNER); - if (sendOK) { + if (putOk) { this.brokerController.getBrokerStatsManager().incTopicPutNums(msg.getTopic(), putMessageResult.getAppendMessageResult().getMsgNum(), 1); this.brokerController.getBrokerStatsManager().incTopicPutSize(msg.getTopic(), putMessageResult.getAppendMessageResult().getWroteBytes()); this.brokerController.getBrokerStatsManager().incBrokerPutNums(putMessageResult.getAppendMessageResult().getMsgNum()); - response.setRemark(null); + responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); responseHeader.setQueueId(queueIdInt); responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); @@ -315,4 +313,30 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen } } } + + class PushReplyResult { + boolean pushOk; + String remark; + + public PushReplyResult(boolean pushOk) { + this.pushOk = pushOk; + remark = ""; + } + + public boolean isPushOk() { + return pushOk; + } + + public void setPushOk(boolean pushOk) { + this.pushOk = pushOk; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + } } From cc7543ff756c535df0e9d1b395563a2be1e4d1ef Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Tue, 24 Sep 2019 21:28:49 +0800 Subject: [PATCH 14/73] add reply interface to DefaultLitePullConsumer --- .../consumer/DefaultLitePullConsumer.java | 51 +++++++++++++++++++ .../consumer/DefaultLitePullConsumerImpl.java | 25 +++++++++ .../client/utils/MessageUtilsTest.java | 2 +- 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 99976d55d3..460558e407 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -22,13 +22,18 @@ import java.util.List; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingException; public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { @@ -411,4 +416,50 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) { this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis; } + + /** + * send a reply message to the producer of the original request message + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param timeoutMillis + * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + return this.defaultLitePullConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message asynchronously + * @param requestMsg original request message + * @param replyContent contents of reply message + * @param replyCallback callback to execute on replying completion. + * @param timeoutMillis + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultLitePullConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); + } + + /** + * send a reply message to the producer of the original request message oneway + * @param requestMsg original request message + * @param replyContent contents of reply message + * @throws InterruptedException if the thread is interrupted. + * @throws RemotingException if there is any network-tier error. + * @throws MQClientException if there is any client error. + * @throws MQBrokerException if there is any broker error. + */ + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + this.defaultLitePullConsumerImpl.replyOneway(requestMsg, replyContent); + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index a37c3a0149..9308fad7c7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -53,6 +53,9 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -60,6 +63,7 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; @@ -1070,4 +1074,25 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } + + @Override + public SendResult reply(final Message requestMsg, final byte[] replyContent, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); + } + + @Override + public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, + long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); + } + + @Override + public void replyOneway(final Message requestMsg, + final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { + Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); + this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java index a722fa6cbb..406979c9f8 100644 --- a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java @@ -45,7 +45,7 @@ public class MessageUtilsTest { Message msg = MessageUtil.createReplyMessage(createReplyMessage(null), new byte[] {'a'}); failBecauseExceptionWasNotThrown(MQClientException.class); } catch (MQClientException e) { - assertThat(e).hasMessageContaining("create reply message fail."); + assertThat(e).hasMessageContaining("create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); } } From 777285011e81d145b067a6e73816a6a5c533057f Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 25 Sep 2019 10:31:31 +0800 Subject: [PATCH 15/73] fix unit test err --- .../rocketmq/broker/processor/ReplyMessageProcessorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java index f48594ebbe..a28f780601 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -92,7 +92,6 @@ public class ReplyMessageProcessorTest { Channel mockChannel = mock(Channel.class); when(mockChannel.remoteAddress()).thenReturn(new InetSocketAddress(1024)); when(handlerContext.channel()).thenReturn(mockChannel); - when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); replyMessageProcessor = new ReplyMessageProcessor(brokerController); } From 0c47fcf7164f98bdedb9b35284206eb4eb4e9052 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Thu, 26 Sep 2019 21:19:48 +0800 Subject: [PATCH 16/73] add recommend client configs in rpc mode --- .../apache/rocketmq/client/ClientConfig.java | 16 ++++++++++++++-- .../consumer/DefaultLitePullConsumerImpl.java | 9 +++++++-- .../consumer/DefaultMQPushConsumerImpl.java | 17 +++++++++++------ .../rocketmq/example/rpc/ResponseConsumer.java | 2 ++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index c3e4efa252..d0ae5e1b83 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -53,6 +53,7 @@ public class ClientConfig { * Offset persistent interval for consumer */ private int persistConsumerOffsetInterval = 1000 * 5; + private long pullTimeDelayMillsWhenException = 1000; private boolean unitMode = false; private String unitName; private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "false")); @@ -148,6 +149,7 @@ public class ClientConfig { this.pollNameServerInterval = cc.pollNameServerInterval; this.heartbeatBrokerInterval = cc.heartbeatBrokerInterval; this.persistConsumerOffsetInterval = cc.persistConsumerOffsetInterval; + this.pullTimeDelayMillsWhenException = cc.pullTimeDelayMillsWhenException; this.unitMode = cc.unitMode; this.unitName = cc.unitName; this.vipChannelEnabled = cc.vipChannelEnabled; @@ -165,6 +167,7 @@ public class ClientConfig { cc.pollNameServerInterval = pollNameServerInterval; cc.heartbeatBrokerInterval = heartbeatBrokerInterval; cc.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + cc.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; cc.unitMode = unitMode; cc.unitName = unitName; cc.vipChannelEnabled = vipChannelEnabled; @@ -222,6 +225,14 @@ public class ClientConfig { this.persistConsumerOffsetInterval = persistConsumerOffsetInterval; } + public long getPullTimeDelayMillsWhenException() { + return pullTimeDelayMillsWhenException; + } + + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } + public String getUnitName() { return unitName; } @@ -287,12 +298,13 @@ public class ClientConfig { this.accessChannel = accessChannel; } + @Override public String toString() { return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + instanceName + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + ", pollNameServerInterval=" + pollNameServerInterval - + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" - + persistConsumerOffsetInterval + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + "]"; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 9308fad7c7..d094f661f7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -110,7 +110,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { /** * Delay some time when exception occur */ - private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 1000; + private long pullTimeDelayMillsWhenException = 1000; /** * Flow control interval */ @@ -160,6 +160,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return new Thread(r, "MonitorMessageQueueChangeThread"); } }); + this.pullTimeDelayMillsWhenException = defaultLitePullConsumer.getPullTimeDelayMillsWhenException(); } private void checkServiceState() { @@ -787,7 +788,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); } catch (Throwable e) { - pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION; + pullDelayTimeMills = pullTimeDelayMillsWhenException; log.error("An error occurred in pull message process.", e); } @@ -1075,6 +1076,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } + @Override public SendResult reply(final Message requestMsg, final byte[] replyContent, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index ac92dc9ff5..20c59609ed 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -85,7 +85,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { /** * Delay some time when exception occur */ - private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 3000; + private long pullTimeDelayMillsWhenException = 3000; /** * Flow control interval */ @@ -117,6 +117,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { this.defaultMQPushConsumer = defaultMQPushConsumer; this.rpcHook = rpcHook; + this.pullTimeDelayMillsWhenException = defaultMQPushConsumer.getPullTimeDelayMillsWhenException(); } public void registerFilterMessageHook(final FilterMessageHook hook) { @@ -224,7 +225,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { this.makeSureStateOK(); } catch (MQClientException e) { log.warn("pullMessage exception, consumer state not ok", e); - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); return; } @@ -284,7 +285,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { pullRequest.setNextOffset(offset); } } else { - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); log.info("pull message later because not locked in broker, {}", pullRequest); return; } @@ -292,7 +293,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (null == subscriptionData) { - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); log.warn("find the consumer's subscription failed, {}", pullRequest); return; } @@ -399,7 +400,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { log.warn("execute the pull request exception", e); } - DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); } }; @@ -446,7 +447,7 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { ); } catch (Exception e) { log.error("pullKernelImpl exception", e); - this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); + this.executePullRequestLater(pullRequest, pullTimeDelayMillsWhenException); } } @@ -1171,6 +1172,10 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { } + public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { + this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; + } + @Override public SendResult reply(final Message requestMsg, final byte[] replyContent, long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index a130722a71..f3b3fd1156 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -37,6 +37,8 @@ public class ResponseConsumer { DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup); consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + //recommend client configs + consumer.setPullTimeDelayMillsWhenException(0L); consumer.registerMessageListener(new MessageListenerConcurrently() { @Override From e7b9169b3098c00ac66610d1522910e0ebf2b595 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Sun, 29 Sep 2019 15:10:25 +0800 Subject: [PATCH 17/73] optimize unit test --- .../producer/DefaultMQProducerTest.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 941ee73f42..991a572529 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -27,8 +27,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -319,24 +319,25 @@ public class DefaultMQProducerTest { @Test public void testRequestMessage() throws RemotingException, RequestTimeoutException, MQClientException, InterruptedException, MQBrokerException { when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + final AtomicBoolean finish = new AtomicBoolean(false); new Thread(new Runnable() { @Override public void run() { - boolean flag = true; - ConcurrentHashMap responseMap = null; - while (flag) { - responseMap = RequestFutureTable.getRequestFutureTable(); - if (responseMap != null) { - flag = false; - } - } + ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); assertThat(responseMap).isNotNull(); - for (Map.Entry entry : responseMap.entrySet()) { - RequestResponseFuture future = entry.getValue(); - future.putResponseMessage(message); + while (!finish.get()) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + } + for (Map.Entry entry : responseMap.entrySet()) { + RequestResponseFuture future = entry.getValue(); + future.putResponseMessage(message); + } } } }).start(); Message result = producer.request(message, 3 * 1000L); + finish.getAndSet(true); assertThat(result.getTopic()).isEqualTo("FooBar"); assertThat(result.getBody()).isEqualTo(new byte[] {'a'}); } From 8353551d0cb1fdffab67deb89362d9b70c3679cf Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Mon, 30 Sep 2019 16:39:18 +0800 Subject: [PATCH 18/73] add unit test of reply --- .../consumer/DefaultMQPushConsumerTest.java | 128 ++++++++++++++++++ .../client/utils/MessageUtilsTest.java | 10 ++ 2 files changed, 138 insertions(+) diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index e6f0e86688..a2a469ebfa 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -19,12 +19,16 @@ package org.apache.rocketmq.client.consumer; import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.net.InetSocketAddress; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; @@ -33,6 +37,7 @@ import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientAPIImpl; @@ -46,11 +51,23 @@ import org.apache.rocketmq.client.impl.consumer.PullRequest; import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +import org.apache.rocketmq.client.producer.SendCallback; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Assert; @@ -66,8 +83,10 @@ import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; @@ -123,6 +142,10 @@ public class DefaultMQPushConsumerTest { field.setAccessible(true); field.set(mQClientFactory, mQClientAPIImpl); + field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(mQClientFactory.getDefaultMQProducer().getDefaultMQProducerImpl(), mQClientFactory); + pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); field.setAccessible(true); @@ -256,6 +279,64 @@ public class DefaultMQPushConsumerTest { } } + @Test + public void testReplyMessageSync() throws RemotingException, InterruptedException, MQBrokerException, MQClientException, NoSuchFieldException { + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); + + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + + SendResult sendResult = pushConsumer.reply(createRequestMessage(), new byte[]{'a'}, 3000); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("12345"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + } + + @Test + public void testReplyMessageASync() throws RemotingException, InterruptedException, MQBrokerException, MQClientException, NoSuchFieldException { + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenAnswer(new Answer() { + @Override public Object answer(InvocationOnMock mock) throws Throwable { + CommunicationMode communicationMode = mock.getArgument(5); + SendCallback sendCallback = mock.getArgument(6); + SendResult sendResult = createSendResult(SendStatus.SEND_OK); + switch (communicationMode) { + case SYNC: + return sendCallback; + case ASYNC: + case ONEWAY: + if (sendCallback != null) { + sendCallback.onSuccess(sendResult); + } + } + return null; + } + }); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); + + final CountDownLatch countDownLatch = new CountDownLatch(1); + final AtomicBoolean replySuccess = new AtomicBoolean(false); + pushConsumer.reply(createRequestMessage(), new byte[] {'a'}, new SendCallback() { + @Override public void onSuccess(SendResult sendResult) { + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getOffsetMsgId()).isEqualTo("12345"); + assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + countDownLatch.countDown(); + replySuccess.set(true); + } + + @Override public void onException(Throwable e) { + countDownLatch.countDown(); + } + }, 3000); + countDownLatch.await(3000, TimeUnit.MILLISECONDS); + assertTrue(replySuccess.get()); + } + private DefaultMQPushConsumer createPushConsumer() { DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @@ -294,4 +375,51 @@ public class DefaultMQPushConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + + private SendResult createSendResult(SendStatus sendStatus) { + SendResult sendResult = new SendResult(); + sendResult.setMsgId("12345"); + sendResult.setOffsetMsgId("12345"); + sendResult.setQueueOffset(456); + sendResult.setSendStatus(sendStatus); + sendResult.setRegionId("HZ"); + return sendResult; + } + + public static TopicRouteData createTopicRoute() { + TopicRouteData topicRouteData = new TopicRouteData(); + + topicRouteData.setFilterServerTable(new HashMap>()); + List brokerDataList = new ArrayList(); + BrokerData brokerData = new BrokerData(); + brokerData.setBrokerName("BrokerA"); + brokerData.setCluster("DefaultCluster"); + HashMap brokerAddrs = new HashMap(); + brokerAddrs.put(0L, "127.0.0.1:10911"); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDataList.add(brokerData); + topicRouteData.setBrokerDatas(brokerDataList); + + List queueDataList = new ArrayList(); + QueueData queueData = new QueueData(); + queueData.setBrokerName("BrokerA"); + queueData.setPerm(6); + queueData.setReadQueueNums(3); + queueData.setWriteQueueNums(4); + queueData.setTopicSynFlag(0); + queueDataList.add(queueData); + topicRouteData.setQueueDatas(queueDataList); + + return topicRouteData; + } + + private Message createRequestMessage() { + Message requestMessage = new Message(); + Map map = new HashMap(); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "reply_to_who"); + map.put(MessageConst.PROPERTY_CLUSTER, "DefaultCluster"); + map.put(MessageConst.PROPERTY_MESSAGE_TTL, "3000"); + MessageAccessor.setProperties(requestMessage, map); + return requestMessage; + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java index 406979c9f8..7847e7ee35 100644 --- a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java @@ -49,6 +49,16 @@ public class MessageUtilsTest { } } + @Test + public void testCreateReplyMessage_reqMsgIsNull() throws MQClientException { + try { + Message msg = MessageUtil.createReplyMessage(null, new byte[] {'a'}); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("create reply message fail, requestMessage cannot be null."); + } + } + private Message createReplyMessage(String clusterName) { Message requestMessage = new Message(); Map map = new HashMap(); From df59df379040caabb3d3048d0e4a45ed5306e41a Mon Sep 17 00:00:00 2001 From: huangli Date: Wed, 25 Sep 2019 21:33:48 +0800 Subject: [PATCH 19/73] Add -n (nameServer) option to benchmark consumer; Enable options(-n/-t/w/-s/-c/-r) to TransactionProducer --- .../rocketmq/example/benchmark/Consumer.java | 4 ++ .../benchmark/TransactionProducer.java | 71 ++++++++++++++----- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java index d431d3ecc6..4724a1d7ab 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/Consumer.java @@ -103,6 +103,10 @@ public class Consumer { }, 10000, 10000); DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + if (commandLine.hasOption('n')) { + String ns = commandLine.getOptionValue('n'); + consumer.setNamesrvAddr(ns); + } consumer.setInstanceName(Long.toString(System.currentTimeMillis())); if (filterType == null || expression == null) { diff --git a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java index d9fafdd08e..04707e1a28 100644 --- a/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/benchmark/TransactionProducer.java @@ -24,6 +24,11 @@ import java.util.TimerTask; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.producer.LocalTransactionExecuter; import org.apache.rocketmq.client.producer.LocalTransactionState; @@ -33,20 +38,19 @@ import org.apache.rocketmq.client.producer.TransactionMQProducer; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.srvutil.ServerUtil; public class TransactionProducer { - private static int threadCount; - private static int messageSize; - private static boolean ischeck; - private static boolean ischeckffalse; public static void main(String[] args) throws MQClientException, UnsupportedEncodingException { - threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; - messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; - ischeck = args.length >= 3 && Boolean.parseBoolean(args[2]); - ischeckffalse = args.length >= 4 && Boolean.parseBoolean(args[3]); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + CommandLine commandLine = ServerUtil.parseCmdLine("TransactionProducer", args, buildCommandlineOptions(options), new PosixParser()); - final Message msg = buildMessage(messageSize); + final String topic = commandLine.hasOption('t') ? commandLine.getOptionValue('t').trim() : "BenchmarkTest"; + final int threadCount = commandLine.hasOption('w') ? Integer.parseInt(commandLine.getOptionValue('w')) : 32; + final int messageSize = commandLine.hasOption('s') ? Integer.parseInt(commandLine.getOptionValue('s')) : 2048; + final boolean ischeck = commandLine.hasOption('c') ? Boolean.parseBoolean(commandLine.getOptionValue('c')) : false; + final boolean ischeckffalse = commandLine.hasOption('r') ? Boolean.parseBoolean(commandLine.getOptionValue('r')) : true; final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); @@ -98,6 +102,10 @@ public class TransactionProducer { producer.setInstanceName(Long.toString(System.currentTimeMillis())); producer.setTransactionCheckListener(transactionCheckListener); producer.setDefaultTopicQueueNums(1000); + if (commandLine.hasOption('n')) { + String ns = commandLine.getOptionValue('n'); + producer.setNamesrvAddr(ns); + } producer.start(); final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); @@ -111,7 +119,7 @@ public class TransactionProducer { // Thread.sleep(1000); final long beginTimestamp = System.currentTimeMillis(); SendResult sendResult = - producer.sendMessageInTransaction(msg, tranExecuter, null); + producer.sendMessageInTransaction(buildMessage(messageSize, topic), tranExecuter, null); if (sendResult != null) { statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); @@ -138,18 +146,45 @@ public class TransactionProducer { } } - private static Message buildMessage(final int messageSize) throws UnsupportedEncodingException { - Message msg = new Message(); - msg.setTopic("BenchmarkTest"); + private static Message buildMessage(final int messageSize, String topic) { + try { + Message msg = new Message(); + msg.setTopic(topic); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < messageSize; i += 10) { - sb.append("hello baby"); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 10) { + sb.append("hello baby"); + } + msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + return msg; + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); } + } - msg.setBody(sb.toString().getBytes(RemotingHelper.DEFAULT_CHARSET)); + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("w", "threadCount", true, "Thread count, Default: 32"); + opt.setRequired(false); + options.addOption(opt); - return msg; + opt = new Option("s", "messageSize", true, "Message Size, Default: 2048"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic name, Default: BenchmarkTest"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "check", true, "Check the message, Default: false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("r", "checkResult", true, "Message check result, Default: true"); + opt.setRequired(false); + options.addOption(opt); + + + return options; } } From a9d43fba4a3c5a9284dcb8ba2cbf218f80422e44 Mon Sep 17 00:00:00 2001 From: huangli Date: Fri, 11 Oct 2019 20:09:48 +0800 Subject: [PATCH 20/73] Fix bug that tx message will be checked even it has been committed or rollback. Half message with unknown status may cause a lots of unnecessary check to half messages which send in 6 seconds before check startTime. --- .../broker/transaction/queue/TransactionalMessageBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 84a62761bd..453a335a40 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -141,6 +141,7 @@ public class TransactionalMessageBridge { getMessageResult.getStatus(), topic, group, offset); break; case NO_MESSAGE_IN_QUEUE: + case OFFSET_OVERFLOW_ONE: pullStatus = PullStatus.NO_NEW_MSG; LOGGER.warn("No new message. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", getMessageResult.getStatus(), topic, group, offset); @@ -149,7 +150,6 @@ public class TransactionalMessageBridge { case NO_MATCHED_LOGIC_QUEUE: case OFFSET_FOUND_NULL: case OFFSET_OVERFLOW_BADLY: - case OFFSET_OVERFLOW_ONE: case OFFSET_TOO_SMALL: pullStatus = PullStatus.OFFSET_ILLEGAL; LOGGER.warn("Offset illegal. GetMessageStatus={}, topic={}, groupId={}, requestOffset={}", From 18e44c2bbc11c60e451e548a90dea1bba590497e Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Sat, 12 Oct 2019 16:53:54 +0800 Subject: [PATCH 21/73] remove reply interface in consumer --- .../consumer/DefaultLitePullConsumer.java | 51 ------- .../consumer/DefaultMQPullConsumer.java | 49 ------- .../consumer/DefaultMQPushConsumer.java | 49 ------- .../consumer/DefaultLitePullConsumerImpl.java | 25 ---- .../consumer/DefaultMQPullConsumerImpl.java | 24 ---- .../consumer/DefaultMQPushConsumerImpl.java | 29 +--- .../client/impl/consumer/MQConsumerInner.java | 15 -- .../consumer/DefaultMQPushConsumerTest.java | 128 ------------------ .../example/rpc/ResponseConsumer.java | 19 ++- 9 files changed, 19 insertions(+), 370 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 460558e407..99976d55d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -22,18 +22,13 @@ import java.util.List; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.consumer.store.OffsetStore; -import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.exception.RemotingException; public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { @@ -416,50 +411,4 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) { this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis; } - - /** - * send a reply message to the producer of the original request message - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param timeoutMillis - * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - return this.defaultLitePullConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message asynchronously - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param replyCallback callback to execute on replying completion. - * @param timeoutMillis - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultLitePullConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message oneway - * @param requestMsg original request message - * @param replyContent contents of reply message - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultLitePullConsumerImpl.replyOneway(requestMsg, replyContent); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 9e291b5638..0876a94e4c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -25,10 +25,7 @@ import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -451,50 +448,4 @@ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsume public void setMaxReconsumeTimes(final int maxReconsumeTimes) { this.maxReconsumeTimes = maxReconsumeTimes; } - - /** - * send a reply message to the producer of the original request message - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param timeoutMillis - * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - return this.defaultMQPullConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message asynchronously - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param replyCallback callback to execute on replying completion. - * @param timeoutMillis - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultMQPullConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message oneway - * @param requestMsg original request message - * @param replyContent contents of reply message - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultMQPullConsumerImpl.replyOneway(requestMsg, replyContent); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java index 8aa12a49a2..339f799f9a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -30,15 +30,12 @@ import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.trace.AsyncTraceDispatcher; import org.apache.rocketmq.client.trace.TraceDispatcher; import org.apache.rocketmq.client.trace.hook.ConsumeMessageTraceHookImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; @@ -892,50 +889,4 @@ public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsume public TraceDispatcher getTraceDispatcher() { return traceDispatcher; } - - /** - * send a reply message to the producer of the original request message - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param timeoutMillis - * @return {@link SendResult} instance to inform senders details of the deliverable, say Message ID of the message, - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - return this.defaultMQPushConsumerImpl.reply(requestMsg, replyContent, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message asynchronously - * @param requestMsg original request message - * @param replyContent contents of reply message - * @param replyCallback callback to execute on replying completion. - * @param timeoutMillis - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback replyCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultMQPushConsumerImpl.reply(requestMsg, replyContent, replyCallback, timeoutMillis); - } - - /** - * send a reply message to the producer of the original request message oneway - * @param requestMsg original request message - * @param replyContent contents of reply message - * @throws InterruptedException if the thread is interrupted. - * @throws RemotingException if there is any network-tier error. - * @throws MQClientException if there is any client error. - * @throws MQBrokerException if there is any broker error. - */ - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - this.defaultMQPushConsumerImpl.replyOneway(requestMsg, replyContent); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index d094f661f7..2c673a1b06 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -53,9 +53,6 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -63,7 +60,6 @@ import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; @@ -1079,25 +1075,4 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; } - - @Override - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); - } - - @Override - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); - } - - @Override - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 6c63ff494b..afd72a0800 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -43,9 +43,6 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; @@ -804,25 +801,4 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { public RebalanceImpl getRebalanceImpl() { return rebalanceImpl; } - - @Override - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); - } - - @Override - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); - } - - @Override - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java index 20c59609ed..807e9c6d6f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -27,6 +27,7 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentMap; + import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; @@ -50,22 +51,20 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.stat.ConsumerStatsManager; -import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.ProcessQueueInfo; @@ -76,7 +75,6 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; -import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; @@ -1175,25 +1173,4 @@ public class DefaultMQPushConsumerImpl implements MQConsumerInner { public void setPullTimeDelayMillsWhenException(long pullTimeDelayMillsWhenException) { this.pullTimeDelayMillsWhenException = pullTimeDelayMillsWhenException; } - - @Override - public SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - return this.mQClientFactory.getDefaultMQProducer().send(replyMessage, timeoutMillis); - } - - @Override - public void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().send(replyMessage, sendCallback, timeoutMillis); - } - - @Override - public void replyOneway(final Message requestMsg, - final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException { - Message replyMessage = MessageUtil.createReplyMessage(requestMsg, replyContent); - this.mQClientFactory.getDefaultMQProducer().sendOneway(replyMessage); - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java index 010d1db345..c2e8a1dfc4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/MQConsumerInner.java @@ -17,18 +17,12 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Set; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.remoting.exception.RemotingException; /** * Consumer inner interface @@ -55,13 +49,4 @@ public interface MQConsumerInner { boolean isUnitMode(); ConsumerRunningInfo consumerRunningInfo(); - - SendResult reply(final Message requestMsg, final byte[] replyContent, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; - - void reply(final Message requestMsg, final byte[] replyContent, final SendCallback sendCallback, - long timeoutMillis) throws InterruptedException, RemotingException, MQClientException, MQBrokerException; - - void replyOneway(final Message requestMsg, - final byte[] replyContent) throws RemotingException, MQClientException, InterruptedException; } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java index a2a469ebfa..e6f0e86688 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultMQPushConsumerTest.java @@ -19,16 +19,12 @@ package org.apache.rocketmq.client.consumer; import java.io.ByteArrayOutputStream; import java.lang.reflect.Field; import java.net.InetSocketAddress; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; @@ -37,7 +33,6 @@ import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.hook.SendMessageContext; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQClientAPIImpl; @@ -51,23 +46,11 @@ import org.apache.rocketmq.client.impl.consumer.PullRequest; import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.consumer.RebalancePushImpl; import org.apache.rocketmq.client.impl.factory.MQClientInstance; -import org.apache.rocketmq.client.impl.producer.DefaultMQProducerImpl; -import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; -import org.apache.rocketmq.client.producer.SendCallback; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.client.producer.SendStatus; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageClientExt; -import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; -import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; -import org.apache.rocketmq.common.protocol.route.BrokerData; -import org.apache.rocketmq.common.protocol.route.QueueData; -import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.remoting.exception.RemotingException; import org.junit.After; import org.junit.Assert; @@ -83,10 +66,8 @@ import org.powermock.modules.junit4.PowerMockRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; -import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.nullable; @@ -142,10 +123,6 @@ public class DefaultMQPushConsumerTest { field.setAccessible(true); field.set(mQClientFactory, mQClientAPIImpl); - field = DefaultMQProducerImpl.class.getDeclaredField("mQClientFactory"); - field.setAccessible(true); - field.set(mQClientFactory.getDefaultMQProducer().getDefaultMQProducerImpl(), mQClientFactory); - pullAPIWrapper = spy(new PullAPIWrapper(mQClientFactory, consumerGroup, false)); field = DefaultMQPushConsumerImpl.class.getDeclaredField("pullAPIWrapper"); field.setAccessible(true); @@ -279,64 +256,6 @@ public class DefaultMQPushConsumerTest { } } - @Test - public void testReplyMessageSync() throws RemotingException, InterruptedException, MQBrokerException, MQClientException, NoSuchFieldException { - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); - - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); - - SendResult sendResult = pushConsumer.reply(createRequestMessage(), new byte[]{'a'}, 3000); - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("12345"); - assertThat(sendResult.getQueueOffset()).isEqualTo(456L); - } - - @Test - public void testReplyMessageASync() throws RemotingException, InterruptedException, MQBrokerException, MQClientException, NoSuchFieldException { - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenAnswer(new Answer() { - @Override public Object answer(InvocationOnMock mock) throws Throwable { - CommunicationMode communicationMode = mock.getArgument(5); - SendCallback sendCallback = mock.getArgument(6); - SendResult sendResult = createSendResult(SendStatus.SEND_OK); - switch (communicationMode) { - case SYNC: - return sendCallback; - case ASYNC: - case ONEWAY: - if (sendCallback != null) { - sendCallback.onSuccess(sendResult); - } - } - return null; - } - }); - when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); - - final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicBoolean replySuccess = new AtomicBoolean(false); - pushConsumer.reply(createRequestMessage(), new byte[] {'a'}, new SendCallback() { - @Override public void onSuccess(SendResult sendResult) { - assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); - assertThat(sendResult.getOffsetMsgId()).isEqualTo("12345"); - assertThat(sendResult.getQueueOffset()).isEqualTo(456L); - countDownLatch.countDown(); - replySuccess.set(true); - } - - @Override public void onException(Throwable e) { - countDownLatch.countDown(); - } - }, 3000); - countDownLatch.await(3000, TimeUnit.MILLISECONDS); - assertTrue(replySuccess.get()); - } - private DefaultMQPushConsumer createPushConsumer() { DefaultMQPushConsumer pushConsumer = new DefaultMQPushConsumer(consumerGroup); pushConsumer.registerMessageListener(new MessageListenerConcurrently() { @@ -375,51 +294,4 @@ public class DefaultMQPushConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } - - private SendResult createSendResult(SendStatus sendStatus) { - SendResult sendResult = new SendResult(); - sendResult.setMsgId("12345"); - sendResult.setOffsetMsgId("12345"); - sendResult.setQueueOffset(456); - sendResult.setSendStatus(sendStatus); - sendResult.setRegionId("HZ"); - return sendResult; - } - - public static TopicRouteData createTopicRoute() { - TopicRouteData topicRouteData = new TopicRouteData(); - - topicRouteData.setFilterServerTable(new HashMap>()); - List brokerDataList = new ArrayList(); - BrokerData brokerData = new BrokerData(); - brokerData.setBrokerName("BrokerA"); - brokerData.setCluster("DefaultCluster"); - HashMap brokerAddrs = new HashMap(); - brokerAddrs.put(0L, "127.0.0.1:10911"); - brokerData.setBrokerAddrs(brokerAddrs); - brokerDataList.add(brokerData); - topicRouteData.setBrokerDatas(brokerDataList); - - List queueDataList = new ArrayList(); - QueueData queueData = new QueueData(); - queueData.setBrokerName("BrokerA"); - queueData.setPerm(6); - queueData.setReadQueueNums(3); - queueData.setWriteQueueNums(4); - queueData.setTopicSynFlag(0); - queueDataList.add(queueData); - topicRouteData.setQueueDatas(queueDataList); - - return topicRouteData; - } - - private Message createRequestMessage() { - Message requestMessage = new Message(); - Map map = new HashMap(); - map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "reply_to_who"); - map.put(MessageConst.PROPERTY_CLUSTER, "DefaultCluster"); - map.put(MessageConst.PROPERTY_MESSAGE_TTL, "3000"); - MessageAccessor.setProperties(requestMessage, map); - return requestMessage; - } } diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index f3b3fd1156..39012c8b3f 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -24,20 +24,30 @@ import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.producer.DefaultMQProducer; import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.exception.RemotingException; public class ResponseConsumer { public static void main(String[] args) throws InterruptedException, MQClientException { + String producerGroup = "please_rename_unique_group_name"; String consumerGroup = "please_rename_unique_group_name"; String topic = "RequestTopic"; + // create a producer to send reply message + DefaultMQProducer replyProducer = new DefaultMQProducer(producerGroup); + replyProducer.start(); + + // create consumer DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(consumerGroup); consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); - //recommend client configs + + // recommend client configs consumer.setPullTimeDelayMillsWhenException(0L); consumer.registerMessageListener(new MessageListenerConcurrently() { @@ -48,9 +58,12 @@ public class ResponseConsumer { try { System.out.printf("handle message: %s", msg.toString()); String replyTo = msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); - byte[] replyContent = "reply message contents.".getBytes(); - SendResult replyResult = consumer.reply(msg, replyContent, 3000); + // create reply message with given util, do not create reply message by yourself + Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent); + + // send reply message with producer + SendResult replyResult = replyProducer.send(replyMessage, 3000); System.out.printf("reply to %s , %s %n", replyTo, replyResult.toString()); } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { e.printStackTrace(); From a8c9fe681f01793c4eadb8ed779b1b092fbd7d79 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Sat, 12 Oct 2019 17:30:15 +0800 Subject: [PATCH 22/73] add err code when create reply message exception --- .../org/apache/rocketmq/client/common/ClientErrorCode.java | 1 + .../java/org/apache/rocketmq/client/utils/MessageUtil.java | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java index 633670aecc..bc03b14df2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java +++ b/client/src/main/java/org/apache/rocketmq/client/common/ClientErrorCode.java @@ -24,4 +24,5 @@ public class ClientErrorCode { public static final int NO_NAME_SERVER_EXCEPTION = 10004; public static final int NOT_FOUND_TOPIC_EXCEPTION = 10005; public static final int REQUEST_TIMEOUT_EXCEPTION = 10006; + public static final int CREATE_REPLY_MESSAGE_EXCEPTION = 10007; } \ No newline at end of file diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index 1e975a9a98..6db470fd5d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.client.utils; +import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.Message; @@ -42,9 +43,9 @@ public class MessageUtil { return replyMessage; } else { - throw new MQClientException(-1, "create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); + throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, "create reply message fail, requestMessage error, property[" + MessageConst.PROPERTY_CLUSTER + "] is null."); } } - throw new MQClientException(-1, "create reply message fail, requestMessage cannot be null."); + throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, "create reply message fail, requestMessage cannot be null."); } } From 01efb2dc9a9937475c0680cb648036272263b813 Mon Sep 17 00:00:00 2001 From: von gosling Date: Mon, 14 Oct 2019 10:34:12 +0800 Subject: [PATCH 23/73] docs(README): address core feature description --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1c17c7e242..33b42800cf 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,12 @@ It offers a variety of features: * Pub/Sub messaging model -* Scheduled message delivery +* Financial grade transactional message +* A variety of cross language clients, such as Java, C/C++, Python, Go +* Pluggable transport protocols, such as TCP, SSL, AIO +* Inbuilt message tracing capability, also support opentracing +* Versatile big-data and streaming ecosytem integration * Message retroactivity by time or offset -* Log hub for streaming -* Big data integration * Reliable FIFO and strict ordered messaging in the same queue * Efficient pull&push consumption model * Million-level message accumulation capacity in a single queue @@ -21,9 +23,7 @@ It offers a variety of features: * Various message filter mechanics such as SQL and Tag * Docker images for isolated testing and cloud isolated clusters * Feature-rich administrative dashboard for configuration, metrics and monitoring -* Access control list -* Message trace - +* Authentication and authorisation ---------- From abad01066d885e874fa63ee091b72a25f41120f8 Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Tue, 15 Oct 2019 10:42:18 +0800 Subject: [PATCH 24/73] modify unit test of producer --- .../producer/DefaultMQProducerTest.java | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index b9be49c033..925b532023 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -52,7 +52,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; @@ -103,9 +105,18 @@ public class DefaultMQProducerTest { when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); + any(SendCallback.class), nullable(TopicPublishInfo.class), any(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + Object[] args = invocation.getArguments(); + SendCallback callback = (SendCallback) args[6]; + callback.onSuccess(createSendResult(SendStatus.SEND_OK)); + return new SendResult(); + } + }); } @After @@ -168,6 +179,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -179,7 +191,6 @@ public class DefaultMQProducerTest { @Override public void onException(Throwable e) { - countDownLatch.countDown(); } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); @@ -190,9 +201,12 @@ public class DefaultMQProducerTest { final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(6); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { + cc.incrementAndGet(); + countDownLatch.countDown(); } @Override @@ -227,6 +241,7 @@ public class DefaultMQProducerTest { public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); + when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @Override public void onSuccess(SendResult sendResult) { @@ -238,7 +253,6 @@ public class DefaultMQProducerTest { @Override public void onException(Throwable e) { - countDownLatch.countDown(); } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); From 85e935d78d58c5a74da973a4ab52f58ff4da53e9 Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Tue, 15 Oct 2019 19:47:42 +0800 Subject: [PATCH 25/73] modify unit test --- .../rocketmq/common/message/MessageClientIDSetterTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java index 987f1032fe..e23a80be07 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java @@ -39,7 +39,7 @@ public class MessageClientIDSetterTest { String ipv4HostMsgId = "C0A803CA00002A9F0000000000031367"; String ipv6HostMsgId = "24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"; Calendar cal = Calendar.getInstance(); - cal.set(2019, Calendar.AUGUST, 1); + cal.set(2019, Calendar.OCTOBER, 1); Date date = cal.getTime(); assertThat(MessageClientIDSetter.getNearlyTimeFromID(ipv4HostMsgId)).isCloseTo(date, 24*60*60*1000); assertThat(MessageClientIDSetter.getNearlyTimeFromID(ipv6HostMsgId)).isCloseTo(date, 24*60*60*1000); From ed94087a7b7e63b7a9a3503206a7d5851a677e02 Mon Sep 17 00:00:00 2001 From: Heng Du Date: Tue, 15 Oct 2019 19:48:11 +0800 Subject: [PATCH 26/73] fix(remoting) upgrade fastjson to 1.2.61 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 697a1d3149..0f9c311f56 100644 --- a/pom.xml +++ b/pom.xml @@ -555,7 +555,7 @@ com.alibaba fastjson - 1.2.51 + 1.2.61 org.javassist From ee91ab44bcdd3c1755251e585fa849b75e46e480 Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Tue, 15 Oct 2019 21:00:58 +0800 Subject: [PATCH 27/73] modify unit test of producer --- .../rocketmq/client/producer/DefaultMQProducerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 925b532023..4d4ce0fba3 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -105,7 +105,9 @@ public class DefaultMQProducerTest { when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); - + when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), + nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) + .thenReturn(createSendResult(SendStatus.SEND_OK)); when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), any(SendCallback.class), nullable(TopicPublishInfo.class), any(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) .thenAnswer(new Answer() { From 85a6cc1c8ee97fa6dade53971bc3ab5594707c5a Mon Sep 17 00:00:00 2001 From: wqliang Date: Wed, 16 Oct 2019 10:32:28 +0800 Subject: [PATCH 28/73] fix ha sync transfer timeout --- .../main/java/org/apache/rocketmq/store/ha/HAService.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 84fb4217ea..2d02d43169 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -280,9 +280,14 @@ public class HAService { if (!this.requestsRead.isEmpty()) { for (CommitLog.GroupCommitRequest req : this.requestsRead) { boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); - for (int i = 0; !transferOK && i < 5; i++) { + long waitUntillWhen = HAService.this.defaultMessageStore.getSystemClock().now() + + HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(); + while (HAService.this.defaultMessageStore.getSystemClock().now() < waitUntillWhen) { this.notifyTransferObject.waitForRunning(1000); transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); + if (transferOK) { + break; + } } if (!transferOK) { From 450dda3ed5e5e8af5680d80632da0918179eae8c Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Wed, 16 Oct 2019 15:16:49 +0800 Subject: [PATCH 29/73] modify unit test of producer --- .../client/producer/DefaultMQProducerTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 4d4ce0fba3..f6208783e6 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -181,6 +181,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); + final AtomicInteger cc = new AtomicInteger(0); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override @@ -189,6 +190,7 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); + cc.incrementAndGet(); } @Override @@ -196,6 +198,7 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); } @Test @@ -207,7 +210,6 @@ public class DefaultMQProducerTest { SendCallback sendCallback = new SendCallback() { @Override public void onSuccess(SendResult sendResult) { - cc.incrementAndGet(); countDownLatch.countDown(); } @@ -229,19 +231,21 @@ public class DefaultMQProducerTest { message.setTopic("test"); message.setBody("hello world".getBytes()); producer.send(new Message(), sendCallback); - producer.send(message, sendCallback, 1000); producer.send(message, new MessageQueue(), sendCallback); producer.send(new Message(), new MessageQueue(), sendCallback, 1000); producer.send(new Message(), messageQueueSelector, null, sendCallback); producer.send(message, messageQueueSelector, null, sendCallback, 1000); + //this message is send success + producer.send(message, sendCallback, 1000); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(6); + assertThat(cc.get()).isEqualTo(5); } @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(1); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @@ -251,6 +255,7 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); + cc.incrementAndGet(); } @Override @@ -258,6 +263,7 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); } @Test From ebc0ede8593c1e4316cfa0b5b28a8eb884efb312 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 16 Oct 2019 15:34:17 +0800 Subject: [PATCH 30/73] optimization exception declare of request --- .../impl/producer/DefaultMQProducerImpl.java | 108 ++++++++---------- .../client/producer/DefaultMQProducer.java | 12 +- .../rocketmq/client/producer/MQProducer.java | 8 +- 3 files changed, 61 insertions(+), 67 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index e740c834b4..c2736bc0cb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -1377,32 +1377,28 @@ public class DefaultMQProducerImpl implements MQProducerInner { } } - public void request(Message msg, final RequestCallback requestCallback, long timeout) throws RemotingException { + public void request(Message msg, final RequestCallback requestCallback, long timeout) + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); - try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); - long cost = System.currentTimeMillis() - beginTimestamp; - this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { - @Override - public void onSuccess(SendResult sendResult) { - requestResponseFuture.setSendReqeustOk(true); - } + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } - @Override - public void onException(Throwable e) { - requestResponseFuture.setCause(e); - requestFail(requestUniqId); - } - }, timeout - cost); - } catch (Exception ex) { - log.warn("send request message to <{}> failed.", msg.getTopic(), ex); - throw new RemotingSendRequestException(msg.getTopic(), ex); - } + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, timeout - cost); } public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, @@ -1447,32 +1443,29 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void request(final Message msg, final MessageQueueSelector selector, final Object arg, - final RequestCallback requestCallback, final long timeout) throws RemotingException { + final RequestCallback requestCallback, final long timeout) + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); - try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); - long cost = System.currentTimeMillis() - beginTimestamp; - this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { - @Override - public void onSuccess(SendResult sendResult) { - requestResponseFuture.setSendReqeustOk(true); - } + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } + + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, timeout - cost); - @Override - public void onException(Throwable e) { - requestResponseFuture.setCause(e); - requestFail(requestUniqId); - } - }, timeout - cost); - } catch (Exception ex) { - log.warn("send request message to <{}> failed.", msg.getTopic(), ex); - throw new RemotingSendRequestException(msg.getTopic(), ex); - } } public Message request(final Message msg, final MessageQueue mq, final long timeout) @@ -1516,32 +1509,27 @@ public class DefaultMQProducerImpl implements MQProducerInner { } public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws RemotingException { + throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); - try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); - long cost = System.currentTimeMillis() - beginTimestamp; - this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { - @Override - public void onSuccess(SendResult sendResult) { - requestResponseFuture.setSendReqeustOk(true); - } + long cost = System.currentTimeMillis() - beginTimestamp; + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { + @Override + public void onSuccess(SendResult sendResult) { + requestResponseFuture.setSendReqeustOk(true); + } - @Override - public void onException(Throwable e) { - requestResponseFuture.setCause(e); - requestFail(requestUniqId); - } - }, null, timeout - cost); - } catch (Exception ex) { - log.warn("send request message to <{}> failed.", msg.getTopic(), ex); - throw new RemotingSendRequestException(msg.getTopic(), ex); - } + @Override + public void onException(Throwable e) { + requestResponseFuture.setCause(e); + requestFail(requestUniqId); + } + }, null, timeout - cost); } private void requestFail(final String requestUniqId) { diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java index fdb489eebd..faa79f54c9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/DefaultMQProducer.java @@ -579,6 +579,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws RemotingException if there is any network-tier error. * @throws MQBrokerException if there is any broker error. * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. */ @Override public Message request(final Message msg, final long timeout) throws RequestTimeoutException, MQClientException, @@ -601,10 +602,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws MQClientException if there is any client error. * @throws RemotingException if there is any network-tier error. * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. */ @Override public void request(final Message msg, final RequestCallback requestCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException, MQBrokerException, RequestTimeoutException { + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, requestCallback, timeout); } @@ -621,6 +623,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws RemotingException if there is any network-tier error. * @throws MQBrokerException if there is any broker error. * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. */ @Override public Message request(final Message msg, final MessageQueueSelector selector, final Object arg, @@ -641,11 +644,12 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws MQClientException if there is any client error. * @throws RemotingException if there is any network-tier error. * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. */ @Override public void request(final Message msg, final MessageQueueSelector selector, final Object arg, final RequestCallback requestCallback, final long timeout) throws MQClientException, RemotingException, - InterruptedException { + InterruptedException, MQBrokerException { msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, selector, arg, requestCallback, timeout); } @@ -660,6 +664,7 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws RemotingException if there is any network-tier error. * @throws MQBrokerException if there is any broker error. * @throws InterruptedException if the thread is interrupted. + * @throws RequestTimeoutException if request timeout. */ @Override public Message request(final Message msg, final MessageQueue mq, final long timeout) @@ -678,10 +683,11 @@ public class DefaultMQProducer extends ClientConfig implements MQProducer { * @throws MQClientException if there is any client error. * @throws RemotingException if there is any network-tier error. * @throws InterruptedException if the thread is interrupted. + * @throws MQBrokerException if there is any broker error. */ @Override public void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws MQClientException, RemotingException, InterruptedException { + throws MQClientException, RemotingException, InterruptedException, MQBrokerException { msg.setTopic(withNamespace(msg.getTopic())); this.defaultMQProducerImpl.request(msg, mq, requestCallback, timeout); } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java index a837c3d232..c6cf4c9359 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/MQProducer.java @@ -105,7 +105,7 @@ public interface MQProducer extends MQAdmin { RemotingException, MQBrokerException, InterruptedException; void request(final Message msg, final RequestCallback requestCallback, final long timeout) - throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException, MQBrokerException; + throws MQClientException, RemotingException, InterruptedException, MQBrokerException; Message request(final Message msg, final MessageQueueSelector selector, final Object arg, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, @@ -113,12 +113,12 @@ public interface MQProducer extends MQAdmin { void request(final Message msg, final MessageQueueSelector selector, final Object arg, final RequestCallback requestCallback, - final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, - InterruptedException; + final long timeout) throws MQClientException, RemotingException, + InterruptedException, MQBrokerException; Message request(final Message msg, final MessageQueue mq, final long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException; void request(final Message msg, final MessageQueue mq, final RequestCallback requestCallback, long timeout) - throws RequestTimeoutException, MQClientException, RemotingException, InterruptedException; + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; } From 9800afb002a9ad28244e3e5a04494b70b4a2c17f Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 16 Oct 2019 16:06:35 +0800 Subject: [PATCH 31/73] rename REQUEST_UNIQ_ID to CORRELATION_ID --- .../client/impl/ClientRemotingProcessor.java | 8 +-- .../impl/producer/DefaultMQProducerImpl.java | 59 +++++++++---------- .../client/producer/RequestFutureTable.java | 2 +- .../producer/RequestResponseFuture.java | 10 ++-- .../rocketmq/client/utils/MessageUtil.java | 4 +- .../rocketmq/common/message/MessageConst.java | 4 +- ...uestIdUtil.java => CorrelationIdUtil.java} | 4 +- 7 files changed, 45 insertions(+), 46 deletions(-) rename common/src/main/java/org/apache/rocketmq/common/utils/{RequestIdUtil.java => CorrelationIdUtil.java} (91%) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 3bea2d8e5d..5cac5e832a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -280,12 +280,12 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { } private void processReplyMessage(MessageExt replyMsg) { - final String uniqueId = replyMsg.getUserProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); - final RequestResponseFuture requestResponseFuture = RequestFutureTable.getRequestFutureTable().get(uniqueId); + final String correlationId = replyMsg.getUserProperty(MessageConst.PROPERTY_CORRELATION_ID); + final RequestResponseFuture requestResponseFuture = RequestFutureTable.getRequestFutureTable().get(correlationId); if (requestResponseFuture != null) { requestResponseFuture.putResponseMessage(replyMsg); - RequestFutureTable.getRequestFutureTable().remove(uniqueId); + RequestFutureTable.getRequestFutureTable().remove(correlationId); if (requestResponseFuture.getRequestCallback() != null) { requestResponseFuture.getRequestCallback().onSuccess(replyMsg); @@ -293,7 +293,7 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { requestResponseFuture.putResponseMessage(replyMsg); } } else { - log.warn(String.format("receive reply message, but not matched any request, REQUEST_UNIQ_ID: %s", uniqueId)); + log.warn(String.format("receive reply message, but not matched any request, CorrelationId: %s", correlationId)); } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index c2736bc0cb..c1c37737d8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -85,13 +85,12 @@ import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHe import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; -import org.apache.rocketmq.common.utils.RequestIdUtil; +import org.apache.rocketmq.common.utils.CorrelationIdUtil; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingConnectException; import org.apache.rocketmq.remoting.exception.RemotingException; -import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; @@ -1341,11 +1340,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { long timeout) throws RequestTimeoutException, MQClientException, RemotingException, MQBrokerException, InterruptedException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { @@ -1373,7 +1372,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } return responseMessage; } finally { - RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + RequestFutureTable.getRequestFutureTable().remove(correlationId); } } @@ -1381,10 +1380,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendDefaultImpl(msg, CommunicationMode.ASYNC, new SendCallback() { @@ -1396,7 +1395,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { @Override public void onException(Throwable e) { requestResponseFuture.setCause(e); - requestFail(requestUniqId); + requestFail(correlationId); } }, timeout - cost); } @@ -1406,11 +1405,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { InterruptedException, RequestTimeoutException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { @@ -1438,7 +1437,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } return responseMessage; } finally { - RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + RequestFutureTable.getRequestFutureTable().remove(correlationId); } } @@ -1447,10 +1446,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, new SendCallback() { @@ -1462,7 +1461,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { @Override public void onException(Throwable e) { requestResponseFuture.setCause(e); - requestFail(requestUniqId); + requestFail(correlationId); } }, timeout - cost); @@ -1472,11 +1471,11 @@ public class DefaultMQProducerImpl implements MQProducerInner { throws MQClientException, RemotingException, MQBrokerException, InterruptedException, RequestTimeoutException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); try { - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, null); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, null); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { @@ -1504,7 +1503,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { } return responseMessage; } finally { - RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + RequestFutureTable.getRequestFutureTable().remove(correlationId); } } @@ -1512,10 +1511,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { throws RemotingException, InterruptedException, MQClientException, MQBrokerException { long beginTimestamp = System.currentTimeMillis(); prepareSendRequest(msg, timeout); - final String requestUniqId = msg.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + final String correlationId = msg.getProperty(MessageConst.PROPERTY_CORRELATION_ID); - final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(requestUniqId, timeout, requestCallback); - RequestFutureTable.getRequestFutureTable().put(requestUniqId, requestResponseFuture); + final RequestResponseFuture requestResponseFuture = new RequestResponseFuture(correlationId, timeout, requestCallback); + RequestFutureTable.getRequestFutureTable().put(correlationId, requestResponseFuture); long cost = System.currentTimeMillis() - beginTimestamp; this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, new SendCallback() { @@ -1527,13 +1526,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { @Override public void onException(Throwable e) { requestResponseFuture.setCause(e); - requestFail(requestUniqId); + requestFail(correlationId); } }, null, timeout - cost); } - private void requestFail(final String requestUniqId) { - RequestResponseFuture responseFuture = RequestFutureTable.getRequestFutureTable().remove(requestUniqId); + private void requestFail(final String correlationId) { + RequestResponseFuture responseFuture = RequestFutureTable.getRequestFutureTable().remove(correlationId); if (responseFuture != null) { responseFuture.setSendReqeustOk(false); responseFuture.putResponseMessage(null); @@ -1546,9 +1545,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { } private void prepareSendRequest(final Message msg, long timeout) { - String requestUniqId = RequestIdUtil.createUniqueRequestId(); + String correlationId = CorrelationIdUtil.createCorrelationId(); String requestClientId = this.getmQClientFactory().getClientId(); - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_CORRELATION_ID, correlationId); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO, requestClientId); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TTL, String.valueOf(timeout)); diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java index 6ce53119ea..3d4caa208b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestFutureTable.java @@ -45,7 +45,7 @@ public class RequestFutureTable { if (rep.isTimeout()) { it.remove(); rfList.add(rep); - log.warn("remove timeout request, REQUEST_UNIQ_ID={}" + rep.getRequestUniqId()); + log.warn("remove timeout request, CorrelationId={}" + rep.getCorrelationId()); } } diff --git a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java index c476ba30f1..c54b236b16 100644 --- a/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java +++ b/client/src/main/java/org/apache/rocketmq/client/producer/RequestResponseFuture.java @@ -22,7 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.rocketmq.common.message.Message; public class RequestResponseFuture { - private final String requestUniqId; + private final String correlationId; private final RequestCallback requestCallback; private final long beginTimestamp = System.currentTimeMillis(); private final Message requestMsg = null; @@ -32,8 +32,8 @@ public class RequestResponseFuture { private volatile boolean sendRequestOk = true; private volatile Throwable cause = null; - public RequestResponseFuture(String requestUniqId, long timeoutMillis, RequestCallback requestCallback) { - this.requestUniqId = requestUniqId; + public RequestResponseFuture(String correlationId, long timeoutMillis, RequestCallback requestCallback) { + this.correlationId = correlationId; this.timeoutMillis = timeoutMillis; this.requestCallback = requestCallback; } @@ -63,8 +63,8 @@ public class RequestResponseFuture { this.countDownLatch.countDown(); } - public String getRequestUniqId() { - return requestUniqId; + public String getCorrelationId() { + return correlationId; } public long getTimeoutMillis() { diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index 6db470fd5d..626b4525bb 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -30,14 +30,14 @@ public class MessageUtil { Message replyMessage = new Message(); String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); - String requestUniqId = requestMessage.getProperty(MessageConst.PROPERTY_REQUEST_UNIQ_ID); + String correlationId = requestMessage.getProperty(MessageConst.PROPERTY_CORRELATION_ID); String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); replyMessage.setBody(body); if (cluster != null) { String replyTopic = MixAll.getReplyTopic(cluster); replyMessage.setTopic(replyTopic); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_REQUEST_UNIQ_ID, requestUniqId); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_CORRELATION_ID, correlationId); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO, replyTo); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index 178e417c7a..f773c2b945 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -45,7 +45,7 @@ public class MessageConst { public static final String PROPERTY_TRANSACTION_CHECK_TIMES = "TRANSACTION_CHECK_TIMES"; public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String PROPERTY_INSTANCE_ID = "INSTANCE_ID"; - public static final String PROPERTY_REQUEST_UNIQ_ID = "REQUEST_UNIQ_ID"; + public static final String PROPERTY_CORRELATION_ID = "CORRELATION_ID"; public static final String PROPERTY_MESSAGE_REPLY_TO = "REPLY_TO"; public static final String PROPERTY_MESSAGE_TTL = "TTL"; public static final String PROPERTY_REPLY_MESSAGE_ARRIVE_TIME = "ARRIVE_TIME"; @@ -81,7 +81,7 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_MAX_RECONSUME_TIMES); STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); STRING_HASH_SET.add(PROPERTY_INSTANCE_ID); - STRING_HASH_SET.add(PROPERTY_REQUEST_UNIQ_ID); + STRING_HASH_SET.add(PROPERTY_CORRELATION_ID); STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO); STRING_HASH_SET.add(PROPERTY_MESSAGE_TTL); STRING_HASH_SET.add(PROPERTY_REPLY_MESSAGE_ARRIVE_TIME); diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java similarity index 91% rename from common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java rename to common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java index 86c5096df5..86d1fd4d42 100644 --- a/common/src/main/java/org/apache/rocketmq/common/utils/RequestIdUtil.java +++ b/common/src/main/java/org/apache/rocketmq/common/utils/CorrelationIdUtil.java @@ -19,8 +19,8 @@ package org.apache.rocketmq.common.utils; import java.util.UUID; -public class RequestIdUtil { - public static String createUniqueRequestId() { +public class CorrelationIdUtil { + public static String createCorrelationId() { return UUID.randomUUID().toString(); } } From f16fd08c5e39101d93d83febeb29cbbddd8434e6 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 16 Oct 2019 17:15:11 +0800 Subject: [PATCH 32/73] rename REPLY_TO to REPLY_TO_CLIENT --- .../broker/processor/ReplyMessageProcessor.java | 6 +++--- .../broker/processor/ReplyMessageProcessorTest.java | 4 +--- .../client/impl/producer/DefaultMQProducerImpl.java | 2 +- .../apache/rocketmq/client/utils/MessageUtil.java | 8 ++++++-- .../rocketmq/client/utils/MessageUtilsTest.java | 12 ++++++++++-- .../apache/rocketmq/common/message/MessageConst.java | 4 ++-- .../rocketmq/example/rpc/ResponseConsumer.java | 2 +- 7 files changed, 24 insertions(+), 14 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java index 265c5c122f..565857a347 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java @@ -173,7 +173,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT, replyMessageRequestHeader); request.setBody(msg.getBody()); - String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + String senderId = msg.getProperties().get(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); PushReplyResult pushReplyResult = new PushReplyResult(false); if (senderId != null) { @@ -207,9 +207,9 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor implemen log.warn(pushReplyResult.getRemark()); } } else { - log.warn("REPLY_TO is null, can not reply message"); + log.warn(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + " is null, can not reply message"); pushReplyResult.setPushOk(false); - pushReplyResult.setRemark("reply message properties[" + MessageConst.PROPERTY_MESSAGE_REPLY_TO + "] is null"); + pushReplyResult.setRemark("reply message properties[" + MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT + "] is null"); } return pushReplyResult; } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java index a28f780601..94fd8f560d 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -30,7 +30,6 @@ import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; @@ -58,7 +57,6 @@ import org.mockito.junit.MockitoJUnitRunner; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -126,7 +124,7 @@ public class ReplyMessageProcessorTest { requestHeader.setFlag(124); requestHeader.setReconsumeTimes(0); Map map = new HashMap(); - map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "127.0.0.1"); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, "127.0.0.1"); requestHeader.setProperties(MessageDecoder.messageProperties2String(map)); return requestHeader; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index c1c37737d8..f2cc605b97 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -1548,7 +1548,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { String correlationId = CorrelationIdUtil.createCorrelationId(); String requestClientId = this.getmQClientFactory().getClientId(); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_CORRELATION_ID, correlationId); - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO, requestClientId); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, requestClientId); MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MESSAGE_TTL, String.valueOf(timeout)); boolean hasRouteData = this.getmQClientFactory().getTopicRouteTable().containsKey(msg.getTopic()); diff --git a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java index 626b4525bb..416ba44da3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java +++ b/client/src/main/java/org/apache/rocketmq/client/utils/MessageUtil.java @@ -29,7 +29,7 @@ public class MessageUtil { if (requestMessage != null) { Message replyMessage = new Message(); String cluster = requestMessage.getProperty(MessageConst.PROPERTY_CLUSTER); - String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + String replyTo = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); String correlationId = requestMessage.getProperty(MessageConst.PROPERTY_CORRELATION_ID); String ttl = requestMessage.getProperty(MessageConst.PROPERTY_MESSAGE_TTL); replyMessage.setBody(body); @@ -38,7 +38,7 @@ public class MessageUtil { replyMessage.setTopic(replyTopic); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TYPE, MixAll.REPLY_MESSAGE_FLAG); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_CORRELATION_ID, correlationId); - MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO, replyTo); + MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, replyTo); MessageAccessor.putProperty(replyMessage, MessageConst.PROPERTY_MESSAGE_TTL, ttl); return replyMessage; @@ -48,4 +48,8 @@ public class MessageUtil { } throw new MQClientException(ClientErrorCode.CREATE_REPLY_MESSAGE_EXCEPTION, "create reply message fail, requestMessage cannot be null."); } + + public static String getReplyToClient(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT); + } } diff --git a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java index 7847e7ee35..803e596fc8 100644 --- a/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/utils/MessageUtilsTest.java @@ -35,7 +35,7 @@ public class MessageUtilsTest { public void testCreateReplyMessage() throws MQClientException { Message msg = MessageUtil.createReplyMessage(createReplyMessage("clusterName"), new byte[] {'a'}); assertThat(msg.getTopic()).isEqualTo("clusterName" + "_" + MixAll.REPLY_TOPIC_POSTFIX); - assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO)).isEqualTo("127.0.0.1"); + assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT)).isEqualTo("127.0.0.1"); assertThat(msg.getProperty(MessageConst.PROPERTY_MESSAGE_TTL)).isEqualTo("3000"); } @@ -59,10 +59,18 @@ public class MessageUtilsTest { } } + @Test + public void testGetReplyToClient() throws MQClientException { + Message msg = createReplyMessage("clusterName"); + String replyToClient = MessageUtil.getReplyToClient(msg); + assertThat(replyToClient).isNotNull(); + assertThat(replyToClient).isEqualTo("127.0.0.1"); + } + private Message createReplyMessage(String clusterName) { Message requestMessage = new Message(); Map map = new HashMap(); - map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO, "127.0.0.1"); + map.put(MessageConst.PROPERTY_MESSAGE_REPLY_TO_CLIENT, "127.0.0.1"); map.put(MessageConst.PROPERTY_CLUSTER, clusterName); map.put(MessageConst.PROPERTY_MESSAGE_TTL, "3000"); MessageAccessor.setProperties(requestMessage, map); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index f773c2b945..5bdc846562 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -46,7 +46,7 @@ public class MessageConst { public static final String PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS = "CHECK_IMMUNITY_TIME_IN_SECONDS"; public static final String PROPERTY_INSTANCE_ID = "INSTANCE_ID"; public static final String PROPERTY_CORRELATION_ID = "CORRELATION_ID"; - public static final String PROPERTY_MESSAGE_REPLY_TO = "REPLY_TO"; + public static final String PROPERTY_MESSAGE_REPLY_TO_CLIENT = "REPLY_TO_CLIENT"; public static final String PROPERTY_MESSAGE_TTL = "TTL"; public static final String PROPERTY_REPLY_MESSAGE_ARRIVE_TIME = "ARRIVE_TIME"; public static final String PROPERTY_PUSH_REPLY_TIME = "PUSH_REPLY_TIME"; @@ -82,7 +82,7 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_CONSUME_START_TIMESTAMP); STRING_HASH_SET.add(PROPERTY_INSTANCE_ID); STRING_HASH_SET.add(PROPERTY_CORRELATION_ID); - STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO); + STRING_HASH_SET.add(PROPERTY_MESSAGE_REPLY_TO_CLIENT); STRING_HASH_SET.add(PROPERTY_MESSAGE_TTL); STRING_HASH_SET.add(PROPERTY_REPLY_MESSAGE_ARRIVE_TIME); STRING_HASH_SET.add(PROPERTY_PUSH_REPLY_TIME); diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index 39012c8b3f..5a53538b54 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -57,7 +57,7 @@ public class ResponseConsumer { for (MessageExt msg : msgs) { try { System.out.printf("handle message: %s", msg.toString()); - String replyTo = msg.getProperty(MessageConst.PROPERTY_MESSAGE_REPLY_TO); + String replyTo = MessageUtil.getReplyToClient(msg); byte[] replyContent = "reply message contents.".getBytes(); // create reply message with given util, do not create reply message by yourself Message replyMessage = MessageUtil.createReplyMessage(msg, replyContent); From 4fa263f75674ebeaf51a2e654af600cfefb599e3 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 16 Oct 2019 17:34:56 +0800 Subject: [PATCH 33/73] code style format --- .../java/org/apache/rocketmq/example/rpc/ResponseConsumer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java index 5a53538b54..c62c7d4eb6 100644 --- a/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/rpc/ResponseConsumer.java @@ -29,7 +29,6 @@ import org.apache.rocketmq.client.producer.SendResult; import org.apache.rocketmq.client.utils.MessageUtil; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.exception.RemotingException; From b6e4be83c7a4bddb56eedadd01e0a344ea429ff1 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Wed, 16 Oct 2019 21:22:09 +0800 Subject: [PATCH 34/73] fix unit test --- .../rocketmq/client/producer/DefaultMQProducerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 991a572529..3811753573 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -398,13 +398,13 @@ public class DefaultMQProducerTest { try { producer.request(message, requestCallback, 3 * 1000L); - failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); - } catch (RemotingSendRequestException e) { + failBecauseExceptionWasNotThrown(Exception.class); + } catch (Exception e) { ConcurrentHashMap responseMap = RequestFutureTable.getRequestFutureTable(); assertThat(responseMap).isNotNull(); for (Map.Entry entry : responseMap.entrySet()) { RequestResponseFuture future = entry.getValue(); - future.getRequestCallback().onException(new Throwable()); + future.getRequestCallback().onException(e); } } countDownLatch.await(3000L, TimeUnit.MILLISECONDS); From d9fbb23dd49782651ddb376ffb99b615129aa9dc Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Thu, 17 Oct 2019 08:59:47 +0800 Subject: [PATCH 35/73] modify unit test of producer --- .../rocketmq/client/producer/DefaultMQProducerTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index f6208783e6..2e23eaa7f3 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -244,8 +244,6 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - - final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(1); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @@ -255,7 +253,6 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); - cc.incrementAndGet(); } @Override @@ -263,7 +260,6 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(1); } @Test From 45043e2571609347b37641c1683d14e86eeee873 Mon Sep 17 00:00:00 2001 From: huangying Date: Fri, 18 Oct 2019 12:13:30 +0800 Subject: [PATCH 36/73] =?UTF-8?q?[#1540]=204.5.2=E7=89=88=E6=9C=AC=20?= =?UTF-8?q?=E4=BA=8B=E5=8A=A1=E6=B6=88=E6=81=AF=E8=AE=BE=E7=BD=AE=E5=BB=B6?= =?UTF-8?q?=E8=BF=9F=E6=97=B6=E9=97=B4=E5=90=8E=E5=AF=BC=E8=87=B4RMQ=5FSYS?= =?UTF-8?q?=5FTRANS=5FHALF=5FTOPIC=E7=88=86=E6=8E=89=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/producer/DefaultMQProducerImpl.java | 6 ++ .../producer/TransactionMQProducerTest.java | 94 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 62aaef3b13..8e3c331aa4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -1179,6 +1179,12 @@ public class DefaultMQProducerImpl implements MQProducerInner { if (null == localTransactionExecuter && null == transactionListener) { throw new MQClientException("tranExecutor is null", null); } + + // 事务消息屏蔽DelayTimeLevel参数 + if(msg.getDelayTimeLevel() != 0) { + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + } + Validators.checkMessage(msg, this.defaultMQProducer); SendResult sendResult = null; diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java new file mode 100644 index 0000000000..007c15bf5c --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java @@ -0,0 +1,94 @@ +package org.apache.rocketmq.client.producer; + +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.remoting.exception.RemotingException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @program: rocketmq + * @description: 事务消息测试类 + * @author: TonyStark888 + * @create: 2019-10-18 11:22 + **/ +public class TransactionMQProducerTest { + private TransactionMQProducer producer; + private String producerGroupPrefix = "Transaction_PID"; + + @Before + public void init() throws Exception { + String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); + producer = new TransactionMQProducer(producerGroupTemp); + producer.setNamesrvAddr("10.0.133.29:9876"); + producer.setTransactionListener(new DelayTimeLevelTransactionListener()); + producer.start(); + } + + @After + public void terminate() { + producer.shutdown(); + } + + @Test + public void testSendMessage() throws InterruptedException, RemotingException, MQBrokerException { + try { + Message message = new Message("TransactionTopic", "transactionTest", "msg-1", ("HelloTime").getBytes()); + SendResult result = producer.sendMessageInTransaction(message, "HelloTime"); + System.out.printf("Topic:%s send success, misId is:%s%n", message.getTopic(), result.getMsgId()); + + // 挂起5分钟，等待事务回查 + Thread.sleep(5 * 60 * 1000L); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("Transaction Message Send Error"); + } + } + + @Test + public void testSendMessage_DelayTimeLevel() throws RemotingException, InterruptedException, MQBrokerException { + try { + Message message = new Message("TransactionTopic", "transactionTest", "msg-1", ("HelloDelayTime").getBytes()); + message.setDelayTimeLevel(1); + SendResult result = producer.sendMessageInTransaction(message, "HelloDelayTime"); + System.out.printf("Topic:%s send success, misId is:%s%n", message.getTopic(), result.getMsgId()); + + // 挂起5分钟，等待事务回查 + Thread.sleep(5 * 60 * 1000L); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("Transaction Message Send Error"); + } + } +} + +/** + * 简便起见，直接返回Commit + */ +class DelayTimeLevelTransactionListener implements TransactionListener { + /** + * 执行本地事务 + * + * @param message + * @param o + * @return + */ + @Override + public LocalTransactionState executeLocalTransaction(Message message, Object o) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + /** + * 回查本地事务结果 + * + * @param messageExt + * @return + */ + @Override + public LocalTransactionState checkLocalTransaction(MessageExt messageExt) { + return LocalTransactionState.COMMIT_MESSAGE; + } +} From ab09e213aa5d14a4043c858b0631996c0fec9ca7 Mon Sep 17 00:00:00 2001 From: huangying Date: Fri, 18 Oct 2019 14:17:52 +0800 Subject: [PATCH 37/73] [#1540] add comment --- .../rocketmq/client/impl/producer/DefaultMQProducerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 8e3c331aa4..c49853e943 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -1180,7 +1180,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { throw new MQClientException("tranExecutor is null", null); } - // 事务消息屏蔽DelayTimeLevel参数 + // ignore DelayTimeLevel parameter if(msg.getDelayTimeLevel() != 0) { MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); } From 1109d16631e34684654b31e0f36e3656c7f770b5 Mon Sep 17 00:00:00 2001 From: huangying Date: Fri, 18 Oct 2019 14:20:50 +0800 Subject: [PATCH 38/73] [#1540] remove test --- .../producer/TransactionMQProducerTest.java | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java deleted file mode 100644 index 007c15bf5c..0000000000 --- a/client/src/test/java/org/apache/rocketmq/client/producer/TransactionMQProducerTest.java +++ /dev/null @@ -1,94 +0,0 @@ -package org.apache.rocketmq.client.producer; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.remoting.exception.RemotingException; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @program: rocketmq - * @description: 事务消息测试类 - * @author: TonyStark888 - * @create: 2019-10-18 11:22 - **/ -public class TransactionMQProducerTest { - private TransactionMQProducer producer; - private String producerGroupPrefix = "Transaction_PID"; - - @Before - public void init() throws Exception { - String producerGroupTemp = producerGroupPrefix + System.currentTimeMillis(); - producer = new TransactionMQProducer(producerGroupTemp); - producer.setNamesrvAddr("10.0.133.29:9876"); - producer.setTransactionListener(new DelayTimeLevelTransactionListener()); - producer.start(); - } - - @After - public void terminate() { - producer.shutdown(); - } - - @Test - public void testSendMessage() throws InterruptedException, RemotingException, MQBrokerException { - try { - Message message = new Message("TransactionTopic", "transactionTest", "msg-1", ("HelloTime").getBytes()); - SendResult result = producer.sendMessageInTransaction(message, "HelloTime"); - System.out.printf("Topic:%s send success, misId is:%s%n", message.getTopic(), result.getMsgId()); - - // 挂起5分钟，等待事务回查 - Thread.sleep(5 * 60 * 1000L); - } catch (MQClientException e) { - assertThat(e).hasMessageContaining("Transaction Message Send Error"); - } - } - - @Test - public void testSendMessage_DelayTimeLevel() throws RemotingException, InterruptedException, MQBrokerException { - try { - Message message = new Message("TransactionTopic", "transactionTest", "msg-1", ("HelloDelayTime").getBytes()); - message.setDelayTimeLevel(1); - SendResult result = producer.sendMessageInTransaction(message, "HelloDelayTime"); - System.out.printf("Topic:%s send success, misId is:%s%n", message.getTopic(), result.getMsgId()); - - // 挂起5分钟，等待事务回查 - Thread.sleep(5 * 60 * 1000L); - } catch (MQClientException e) { - assertThat(e).hasMessageContaining("Transaction Message Send Error"); - } - } -} - -/** - * 简便起见，直接返回Commit - */ -class DelayTimeLevelTransactionListener implements TransactionListener { - /** - * 执行本地事务 - * - * @param message - * @param o - * @return - */ - @Override - public LocalTransactionState executeLocalTransaction(Message message, Object o) { - return LocalTransactionState.COMMIT_MESSAGE; - } - - /** - * 回查本地事务结果 - * - * @param messageExt - * @return - */ - @Override - public LocalTransactionState checkLocalTransaction(MessageExt messageExt) { - return LocalTransactionState.COMMIT_MESSAGE; - } -} From 1b012a3c88f0ff016030f5013e5a8d85de7c2b76 Mon Sep 17 00:00:00 2001 From: huangying Date: Fri, 18 Oct 2019 14:34:19 +0800 Subject: [PATCH 39/73] [#1540] format checkstyle --- .../impl/producer/DefaultMQProducerImpl.java | 2569 +++++++++-------- 1 file changed, 1285 insertions(+), 1284 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index fc7c0879c7..8346dc7ed2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -34,6 +34,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; + import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -88,1288 +89,1288 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; public class DefaultMQProducerImpl implements MQProducerInner { - private final InternalLogger log = ClientLogger.getLog(); - private final Random random = new Random(); - private final DefaultMQProducer defaultMQProducer; - private final ConcurrentMap topicPublishInfoTable = - new ConcurrentHashMap(); - private final ArrayList sendMessageHookList = new ArrayList(); - private final RPCHook rpcHook; - protected BlockingQueue checkRequestQueue; - protected ExecutorService checkExecutor; - private ServiceState serviceState = ServiceState.CREATE_JUST; - private MQClientInstance mQClientFactory; - private ArrayList checkForbiddenHookList = new ArrayList(); - private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); - - private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); - - private final BlockingQueue asyncSenderThreadPoolQueue; - private final ExecutorService defaultAsyncSenderExecutor; - private ExecutorService asyncSenderExecutor; - - public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { - this(defaultMQProducer, null); - } - - public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { - this.defaultMQProducer = defaultMQProducer; - this.rpcHook = rpcHook; - - this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue(50000); - this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors(), - Runtime.getRuntime().availableProcessors(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.asyncSenderThreadPoolQueue, - new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet()); - } - }); - } - - public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { - this.checkForbiddenHookList.add(checkForbiddenHook); - log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), - checkForbiddenHookList.size()); - } - - public void initTransactionEnv() { - TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; - if (producer.getExecutorService() != null) { - this.checkExecutor = producer.getExecutorService(); - } else { - this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); - this.checkExecutor = new ThreadPoolExecutor( - producer.getCheckThreadPoolMinSize(), - producer.getCheckThreadPoolMaxSize(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.checkRequestQueue); - } - } - - public void destroyTransactionEnv() { - if (this.checkExecutor != null) { - this.checkExecutor.shutdown(); - } - } - - public void registerSendMessageHook(final SendMessageHook hook) { - this.sendMessageHookList.add(hook); - log.info("register sendMessage Hook, {}", hook.hookName()); - } - - public void start() throws MQClientException { - this.start(true); - } - - public void start(final boolean startFactory) throws MQClientException { - switch (this.serviceState) { - case CREATE_JUST: - this.serviceState = ServiceState.START_FAILED; - - this.checkConfig(); - - if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { - this.defaultMQProducer.changeInstanceNameToPID(); - } - - this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); - - boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); - if (!registerOK) { - this.serviceState = ServiceState.CREATE_JUST; - throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() - + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), - null); - } - - this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); - - if (startFactory) { - mQClientFactory.start(); - } - - log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), - this.defaultMQProducer.isSendMessageWithVIPChannel()); - this.serviceState = ServiceState.RUNNING; - break; - case RUNNING: - case START_FAILED: - case SHUTDOWN_ALREADY: - throw new MQClientException("The producer service state not OK, maybe started once, " - + this.serviceState - + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), - null); - default: - break; - } - - this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); - } - - private void checkConfig() throws MQClientException { - Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); - - if (null == this.defaultMQProducer.getProducerGroup()) { - throw new MQClientException("producerGroup is null", null); - } - - if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { - throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", - null); - } - } - - public void shutdown() { - this.shutdown(true); - } - - public void shutdown(final boolean shutdownFactory) { - switch (this.serviceState) { - case CREATE_JUST: - break; - case RUNNING: - this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); - this.defaultAsyncSenderExecutor.shutdown(); - if (shutdownFactory) { - this.mQClientFactory.shutdown(); - } - - log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); - this.serviceState = ServiceState.SHUTDOWN_ALREADY; - break; - case SHUTDOWN_ALREADY: - break; - default: - break; - } - } - - @Override - public Set getPublishTopicList() { - Set topicList = new HashSet(); - for (String key : this.topicPublishInfoTable.keySet()) { - topicList.add(key); - } - - return topicList; - } - - @Override - public boolean isPublishTopicNeedUpdate(String topic) { - TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); - - return null == prev || !prev.ok(); - } - - /** - * This method will be removed in the version 5.0.0 and getCheckListener is recommended. - * - * @return - */ - @Override - @Deprecated - public TransactionCheckListener checkListener() { - if (this.defaultMQProducer instanceof TransactionMQProducer) { - TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionCheckListener(); - } - - return null; - } - - @Override - public TransactionListener getCheckListener() { - if (this.defaultMQProducer instanceof TransactionMQProducer) { - TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionListener(); - } - return null; - } - - @Override - public void checkTransactionState(final String addr, final MessageExt msg, - final CheckTransactionStateRequestHeader header) { - Runnable request = new Runnable() { - private final String brokerAddr = addr; - private final MessageExt message = msg; - private final CheckTransactionStateRequestHeader checkRequestHeader = header; - private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); - - @Override - public void run() { - TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); - TransactionListener transactionListener = getCheckListener(); - if (transactionCheckListener != null || transactionListener != null) { - LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; - Throwable exception = null; - try { - if (transactionCheckListener != null) { - localTransactionState = transactionCheckListener.checkLocalTransactionState(message); - } else if (transactionListener != null) { - log.debug("Used new check API in transaction message"); - localTransactionState = transactionListener.checkLocalTransaction(message); - } else { - log.warn("CheckTransactionState, pick transactionListener by group[{}] failed", group); - } - } catch (Throwable e) { - log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); - exception = e; - } - - this.processTransactionState( - localTransactionState, - group, - exception); - } else { - log.warn("CheckTransactionState, pick transactionCheckListener by group[{}] failed", group); - } - } - - private void processTransactionState( - final LocalTransactionState localTransactionState, - final String producerGroup, - final Throwable exception) { - final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); - thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); - thisHeader.setProducerGroup(producerGroup); - thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); - thisHeader.setFromTransactionCheck(true); - - String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); - if (uniqueKey == null) { - uniqueKey = message.getMsgId(); - } - thisHeader.setMsgId(uniqueKey); - thisHeader.setTransactionId(checkRequestHeader.getTransactionId()); - switch (localTransactionState) { - case COMMIT_MESSAGE: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); - break; - case ROLLBACK_MESSAGE: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); - log.warn("when broker check, client rollback this transaction, {}", thisHeader); - break; - case UNKNOW: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); - log.warn("when broker check, client does not know this transaction state, {}", thisHeader); - break; - default: - break; - } - - String remark = null; - if (exception != null) { - remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); - } - - try { - DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, - 3000); - } catch (Exception e) { - log.error("endTransactionOneway exception", e); - } - } - }; - - this.checkExecutor.submit(request); - } - - @Override - public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { - if (info != null && topic != null) { - TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); - if (prev != null) { - log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); - } - } - } - - @Override - public boolean isUnitMode() { - return this.defaultMQProducer.isUnitMode(); - } - - public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { - createTopic(key, newTopic, queueNum, 0); - } - - public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { - this.makeSureStateOK(); - Validators.checkTopic(newTopic); - - this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); - } - - private void makeSureStateOK() throws MQClientException { - if (this.serviceState != ServiceState.RUNNING) { - throw new MQClientException("The producer service state not OK, " - + this.serviceState - + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), - null); - } - } - - public List fetchPublishMessageQueues(String topic) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); - } - - public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); - } - - public long maxOffset(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); - } - - public long minOffset(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().minOffset(mq); - } - - public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); - } - - public MessageExt viewMessage( - String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - this.makeSureStateOK(); - - return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); - } - - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) - throws MQClientException, InterruptedException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); - } - - public MessageExt queryMessageByUniqKey(String topic, String uniqKey) - throws MQClientException, InterruptedException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); - } - - /** - * DEFAULT ASYNC ------------------------------------------------------- - */ - public void send(Message msg, - SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { - send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws RejectedExecutionException - */ - @Deprecated - public void send(final Message msg, final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); - } catch (Exception e) { - sendCallback.onException(e); - } - } else { - sendCallback.onException( - new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); - } - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("executor rejected ", e); - } - - } - - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { - return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); - } - - public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { - this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); - } - - private void validateNameServerSetting() throws MQClientException { - List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); - if (null == nsList || nsList.isEmpty()) { - throw new MQClientException( - "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); - } - - } - - private SendResult sendDefaultImpl( - Message msg, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final long timeout - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - final long invokeID = random.nextLong(); - long beginTimestampFirst = System.currentTimeMillis(); - long beginTimestampPrev = beginTimestampFirst; - long endTimestamp = beginTimestampFirst; - TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); - if (topicPublishInfo != null && topicPublishInfo.ok()) { - boolean callTimeout = false; - MessageQueue mq = null; - Exception exception = null; - SendResult sendResult = null; - int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; - int times = 0; - String[] brokersSent = new String[timesTotal]; - for (; times < timesTotal; times++) { - String lastBrokerName = null == mq ? null : mq.getBrokerName(); - MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); - if (mqSelected != null) { - mq = mqSelected; - brokersSent[times] = mq.getBrokerName(); - try { - beginTimestampPrev = System.currentTimeMillis(); - if (times > 0) { - //Reset topic with namespace during resend. - msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic())); - } - long costTime = beginTimestampPrev - beginTimestampFirst; - if (timeout < costTime) { - callTimeout = true; - break; - } - - sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); - switch (communicationMode) { - case ASYNC: - return null; - case ONEWAY: - return null; - case SYNC: - if (sendResult.getSendStatus() != SendStatus.SEND_OK) { - if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { - continue; - } - } - - return sendResult; - default: - break; - } - } catch (RemotingException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - continue; - } catch (MQClientException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - continue; - } catch (MQBrokerException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - switch (e.getResponseCode()) { - case ResponseCode.TOPIC_NOT_EXIST: - case ResponseCode.SERVICE_NOT_AVAILABLE: - case ResponseCode.SYSTEM_ERROR: - case ResponseCode.NO_PERMISSION: - case ResponseCode.NO_BUYER_ID: - case ResponseCode.NOT_IN_CURRENT_UNIT: - continue; - default: - if (sendResult != null) { - return sendResult; - } - - throw e; - } - } catch (InterruptedException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); - log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - - log.warn("sendKernelImpl exception", e); - log.warn(msg.toString()); - throw e; - } - } else { - break; - } - } - - if (sendResult != null) { - return sendResult; - } - - String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", - times, - System.currentTimeMillis() - beginTimestampFirst, - msg.getTopic(), - Arrays.toString(brokersSent)); - - info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); - - MQClientException mqClientException = new MQClientException(info, exception); - if (callTimeout) { - throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout"); - } - - if (exception instanceof MQBrokerException) { - mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); - } else if (exception instanceof RemotingConnectException) { - mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION); - } else if (exception instanceof RemotingTimeoutException) { - mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT); - } else if (exception instanceof MQClientException) { - mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION); - } - - throw mqClientException; - } - - validateNameServerSetting(); - - throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), - null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); - } - - private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { - TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); - if (null == topicPublishInfo || !topicPublishInfo.ok()) { - this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - topicPublishInfo = this.topicPublishInfoTable.get(topic); - } - - if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { - return topicPublishInfo; - } else { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); - topicPublishInfo = this.topicPublishInfoTable.get(topic); - return topicPublishInfo; - } - } - - private SendResult sendKernelImpl(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - if (null == brokerAddr) { - tryToFindTopicPublishInfo(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - } - - SendMessageContext context = null; - if (brokerAddr != null) { - brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr); - - byte[] prevBody = msg.getBody(); - try { - //for MessageBatch,ID has been set in the generating process - if (!(msg instanceof MessageBatch)) { - MessageClientIDSetter.setUniqID(msg); - } - - boolean topicWithNamespace = false; - if (null != this.mQClientFactory.getClientConfig().getNamespace()) { - msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace()); - topicWithNamespace = true; - } - - int sysFlag = 0; - boolean msgBodyCompressed = false; - if (this.tryToCompressMessage(msg)) { - sysFlag |= MessageSysFlag.COMPRESSED_FLAG; - msgBodyCompressed = true; - } - - final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { - sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; - } - - if (hasCheckForbiddenHook()) { - CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); - checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); - checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); - checkForbiddenContext.setCommunicationMode(communicationMode); - checkForbiddenContext.setBrokerAddr(brokerAddr); - checkForbiddenContext.setMessage(msg); - checkForbiddenContext.setMq(mq); - checkForbiddenContext.setUnitMode(this.isUnitMode()); - this.executeCheckForbiddenHook(checkForbiddenContext); - } - - if (this.hasSendMessageHook()) { - context = new SendMessageContext(); - context.setProducer(this); - context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - context.setCommunicationMode(communicationMode); - context.setBornHost(this.defaultMQProducer.getClientIP()); - context.setBrokerAddr(brokerAddr); - context.setMessage(msg); - context.setMq(mq); - context.setNamespace(this.defaultMQProducer.getNamespace()); - String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (isTrans != null && isTrans.equals("true")) { - context.setMsgType(MessageType.Trans_Msg_Half); - } - - if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) { - context.setMsgType(MessageType.Delay_Msg); - } - this.executeSendMessageHookBefore(context); - } - - SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); - requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - requestHeader.setTopic(msg.getTopic()); - requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); - requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); - requestHeader.setQueueId(mq.getQueueId()); - requestHeader.setSysFlag(sysFlag); - requestHeader.setBornTimestamp(System.currentTimeMillis()); - requestHeader.setFlag(msg.getFlag()); - requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); - requestHeader.setReconsumeTimes(0); - requestHeader.setUnitMode(this.isUnitMode()); - requestHeader.setBatch(msg instanceof MessageBatch); - if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { - String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); - if (reconsumeTimes != null) { - requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); - } - - String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg); - if (maxReconsumeTimes != null) { - requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); - } - } - - SendResult sendResult = null; - switch (communicationMode) { - case ASYNC: - Message tmpMessage = msg; - boolean messageCloned = false; - if (msgBodyCompressed) { - //If msg body was compressed, msgbody should be reset using prevBody. - //Clone new message using commpressed message body and recover origin massage. - //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 - tmpMessage = MessageAccessor.cloneMessage(msg); - messageCloned = true; - msg.setBody(prevBody); - } - - if (topicWithNamespace) { - if (!messageCloned) { - tmpMessage = MessageAccessor.cloneMessage(msg); - messageCloned = true; - } - msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); - } - - long costTimeAsync = System.currentTimeMillis() - beginStartTime; - if (timeout < costTimeAsync) { - throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); - } - sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( - brokerAddr, - mq.getBrokerName(), - tmpMessage, - requestHeader, - timeout - costTimeAsync, - communicationMode, - sendCallback, - topicPublishInfo, - this.mQClientFactory, - this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(), - context, - this); - break; - case ONEWAY: - case SYNC: - long costTimeSync = System.currentTimeMillis() - beginStartTime; - if (timeout < costTimeSync) { - throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); - } - sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( - brokerAddr, - mq.getBrokerName(), - msg, - requestHeader, - timeout - costTimeSync, - communicationMode, - context, - this); - break; - default: - assert false; - break; - } - - if (this.hasSendMessageHook()) { - context.setSendResult(sendResult); - this.executeSendMessageHookAfter(context); - } - - return sendResult; - } catch (RemotingException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } catch (MQBrokerException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } catch (InterruptedException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } finally { - msg.setBody(prevBody); - msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); - } - } - - throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); - } - - public MQClientInstance getmQClientFactory() { - return mQClientFactory; - } - - private boolean tryToCompressMessage(final Message msg) { - if (msg instanceof MessageBatch) { - //batch dose not support compressing right now - return false; - } - byte[] body = msg.getBody(); - if (body != null) { - if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { - try { - byte[] data = UtilAll.compress(body, zipCompressLevel); - if (data != null) { - msg.setBody(data); - return true; - } - } catch (IOException e) { - log.error("tryToCompressMessage exception", e); - log.warn(msg.toString()); - } - } - } - - return false; - } - - public boolean hasCheckForbiddenHook() { - return !checkForbiddenHookList.isEmpty(); - } - - public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { - if (hasCheckForbiddenHook()) { - for (CheckForbiddenHook hook : checkForbiddenHookList) { - hook.checkForbidden(context); - } - } - } - - public boolean hasSendMessageHook() { - return !this.sendMessageHookList.isEmpty(); - } - - public void executeSendMessageHookBefore(final SendMessageContext context) { - if (!this.sendMessageHookList.isEmpty()) { - for (SendMessageHook hook : this.sendMessageHookList) { - try { - hook.sendMessageBefore(context); - } catch (Throwable e) { - log.warn("failed to executeSendMessageHookBefore", e); - } - } - } - } - - public void executeSendMessageHookAfter(final SendMessageContext context) { - if (!this.sendMessageHookList.isEmpty()) { - for (SendMessageHook hook : this.sendMessageHookList) { - try { - hook.sendMessageAfter(context); - } catch (Throwable e) { - log.warn("failed to executeSendMessageHookAfter", e); - } - } - } - } - - /** - * DEFAULT ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { - try { - this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - /** - * KERNEL SYNC ------------------------------------------------------- - */ - public SendResult send(Message msg, MessageQueue mq) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout()); - } - - public SendResult send(Message msg, MessageQueue mq, long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - if (!msg.getTopic().equals(mq.getTopic())) { - throw new MQClientException("message's topic not equal mq's topic", null); - } - - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout < costTime) { - throw new RemotingTooMuchRequestException("call timeout"); - } - - return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); - } - - /** - * KERNEL ASYNC ------------------------------------------------------- - */ - public void send(Message msg, MessageQueue mq, SendCallback sendCallback) - throws MQClientException, RemotingException, InterruptedException { - send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param mq - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException - */ - @Deprecated - public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - try { - makeSureStateOK(); - Validators.checkMessage(msg, defaultMQProducer); - - if (!msg.getTopic().equals(mq.getTopic())) { - throw new MQClientException("message's topic not equal mq's topic", null); - } - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, - timeout - costTime); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); - } - } catch (Exception e) { - sendCallback.onException(e); - } - - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("executor rejected ", e); - } - - } - - /** - * KERNEL ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg, - MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - try { - this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - /** - * SELECT SYNC ------------------------------------------------------- - */ - public SendResult send(Message msg, MessageQueueSelector selector, Object arg) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout()); - } - - public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout); - } - - private SendResult sendSelectImpl( - Message msg, - MessageQueueSelector selector, - Object arg, - final CommunicationMode communicationMode, - final SendCallback sendCallback, final long timeout - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); - if (topicPublishInfo != null && topicPublishInfo.ok()) { - MessageQueue mq = null; - try { - List messageQueueList = - mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); - Message userMessage = MessageAccessor.cloneMessage(msg); - String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); - userMessage.setTopic(userTopic); - - mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg)); - } catch (Throwable e) { - throw new MQClientException("select message queue throwed exception.", e); - } - - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout < costTime) { - throw new RemotingTooMuchRequestException("sendSelectImpl call timeout"); - } - if (mq != null) { - return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime); - } else { - throw new MQClientException("select message queue return null.", null); - } - } - - validateNameServerSetting(); - throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); - } - - /** - * SELECT ASYNC ------------------------------------------------------- - */ - public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) - throws MQClientException, RemotingException, InterruptedException { - send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param selector - * @param arg - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException - */ - @Deprecated - public void send(final Message msg, final MessageQueueSelector selector, final Object arg, - final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - try { - sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, - timeout - costTime); - } catch (MQBrokerException e) { - throw new MQClientException("unknownn exception", e); - } - } catch (Exception e) { - sendCallback.onException(e); - } - } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); - } - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("exector rejected ", e); - } - } - - /** - * SELECT ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) - throws MQClientException, RemotingException, InterruptedException { - try { - this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter localTransactionExecuter, final Object arg) - throws MQClientException { - TransactionListener transactionListener = getCheckListener(); - if (null == localTransactionExecuter && null == transactionListener) { - throw new MQClientException("tranExecutor is null", null); - } - - // ignore DelayTimeLevel parameter - if(msg.getDelayTimeLevel() != 0) { - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); - } - - Validators.checkMessage(msg, this.defaultMQProducer); - - SendResult sendResult = null; - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup()); - try { - sendResult = this.send(msg); - } catch (Exception e) { - throw new MQClientException("send message Exception", e); - } - - LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; - Throwable localException = null; - switch (sendResult.getSendStatus()) { - case SEND_OK: { - try { - if (sendResult.getTransactionId() != null) { - msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); - } - String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); - if (null != transactionId && !"".equals(transactionId)) { - msg.setTransactionId(transactionId); - } - if (null != localTransactionExecuter) { - localTransactionState = localTransactionExecuter.executeLocalTransactionBranch(msg, arg); - } else if (transactionListener != null) { - log.debug("Used new transaction API"); - localTransactionState = transactionListener.executeLocalTransaction(msg, arg); - } - if (null == localTransactionState) { - localTransactionState = LocalTransactionState.UNKNOW; - } - - if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { - log.info("executeLocalTransactionBranch return {}", localTransactionState); - log.info(msg.toString()); - } - } catch (Throwable e) { - log.info("executeLocalTransactionBranch exception", e); - log.info(msg.toString()); - localException = e; - } - } - break; - case FLUSH_DISK_TIMEOUT: - case FLUSH_SLAVE_TIMEOUT: - case SLAVE_NOT_AVAILABLE: - localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; - break; - default: - break; - } - - try { - this.endTransaction(sendResult, localTransactionState, localException); - } catch (Exception e) { - log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); - } - - TransactionSendResult transactionSendResult = new TransactionSendResult(); - transactionSendResult.setSendStatus(sendResult.getSendStatus()); - transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); - transactionSendResult.setMsgId(sendResult.getMsgId()); - transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); - transactionSendResult.setTransactionId(sendResult.getTransactionId()); - transactionSendResult.setLocalTransactionState(localTransactionState); - return transactionSendResult; - } - - /** - * DEFAULT SYNC ------------------------------------------------------- - */ - public SendResult send( - Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, this.defaultMQProducer.getSendMsgTimeout()); - } - - public void endTransaction( - final SendResult sendResult, - final LocalTransactionState localTransactionState, - final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { - final MessageId id; - if (sendResult.getOffsetMsgId() != null) { - id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); - } else { - id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); - } - String transactionId = sendResult.getTransactionId(); - final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); - EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); - requestHeader.setTransactionId(transactionId); - requestHeader.setCommitLogOffset(id.getOffset()); - switch (localTransactionState) { - case COMMIT_MESSAGE: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); - break; - case ROLLBACK_MESSAGE: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); - break; - case UNKNOW: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); - break; - default: - break; - } - - requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); - requestHeader.setMsgId(sendResult.getMsgId()); - String remark = localException != null ? ("executeLocalTransactionBranch exception: " + localException.toString()) : null; - this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark, - this.defaultMQProducer.getSendMsgTimeout()); - } - - public void setCallbackExecutor(final ExecutorService callbackExecutor) { - this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); - } - - public ExecutorService getAsyncSenderExecutor() { - return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor; - } - - public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) { - this.asyncSenderExecutor = asyncSenderExecutor; - } - - public SendResult send(Message msg, - long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); - } - - public ConcurrentMap getTopicPublishInfoTable() { - return topicPublishInfoTable; - } - - public int getZipCompressLevel() { - return zipCompressLevel; - } - - public void setZipCompressLevel(int zipCompressLevel) { - this.zipCompressLevel = zipCompressLevel; - } - - public ServiceState getServiceState() { - return serviceState; - } - - public void setServiceState(ServiceState serviceState) { - this.serviceState = serviceState; - } - - public long[] getNotAvailableDuration() { - return this.mqFaultStrategy.getNotAvailableDuration(); - } - - public void setNotAvailableDuration(final long[] notAvailableDuration) { - this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration); - } - - public long[] getLatencyMax() { - return this.mqFaultStrategy.getLatencyMax(); - } - - public void setLatencyMax(final long[] latencyMax) { - this.mqFaultStrategy.setLatencyMax(latencyMax); - } - - public boolean isSendLatencyFaultEnable() { - return this.mqFaultStrategy.isSendLatencyFaultEnable(); - } - - public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { - this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable); - } - - public DefaultMQProducer getDefaultMQProducer() { - return defaultMQProducer; - } + private final InternalLogger log = ClientLogger.getLog(); + private final Random random = new Random(); + private final DefaultMQProducer defaultMQProducer; + private final ConcurrentMap topicPublishInfoTable = + new ConcurrentHashMap(); + private final ArrayList sendMessageHookList = new ArrayList(); + private final RPCHook rpcHook; + protected BlockingQueue checkRequestQueue; + protected ExecutorService checkExecutor; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private ArrayList checkForbiddenHookList = new ArrayList(); + private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); + + private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); + + private final BlockingQueue asyncSenderThreadPoolQueue; + private final ExecutorService defaultAsyncSenderExecutor; + private ExecutorService asyncSenderExecutor; + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { + this(defaultMQProducer, null); + } + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { + this.defaultMQProducer = defaultMQProducer; + this.rpcHook = rpcHook; + + this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue(50000); + this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( + Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().availableProcessors(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.asyncSenderThreadPoolQueue, + new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + } + + public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { + this.checkForbiddenHookList.add(checkForbiddenHook); + log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), + checkForbiddenHookList.size()); + } + + public void initTransactionEnv() { + TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; + if (producer.getExecutorService() != null) { + this.checkExecutor = producer.getExecutorService(); + } else { + this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); + this.checkExecutor = new ThreadPoolExecutor( + producer.getCheckThreadPoolMinSize(), + producer.getCheckThreadPoolMaxSize(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.checkRequestQueue); + } + } + + public void destroyTransactionEnv() { + if (this.checkExecutor != null) { + this.checkExecutor.shutdown(); + } + } + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register sendMessage Hook, {}", hook.hookName()); + } + + public void start() throws MQClientException { + this.start(true); + } + + public void start(final boolean startFactory) throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { + this.defaultMQProducer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); + + boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); + + if (startFactory) { + mQClientFactory.start(); + } + + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), + this.defaultMQProducer.isSendMessageWithVIPChannel()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The producer service state not OK, maybe started once, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + + private void checkConfig() throws MQClientException { + Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); + + if (null == this.defaultMQProducer.getProducerGroup()) { + throw new MQClientException("producerGroup is null", null); + } + + if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { + throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", + null); + } + } + + public void shutdown() { + this.shutdown(true); + } + + public void shutdown(final boolean shutdownFactory) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); + this.defaultAsyncSenderExecutor.shutdown(); + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } + + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + @Override + public Set getPublishTopicList() { + Set topicList = new HashSet(); + for (String key : this.topicPublishInfoTable.keySet()) { + topicList.add(key); + } + + return topicList; + } + + @Override + public boolean isPublishTopicNeedUpdate(String topic) { + TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); + + return null == prev || !prev.ok(); + } + + /** + * This method will be removed in the version 5.0.0 and getCheckListener is recommended. + * + * @return + */ + @Override + @Deprecated + public TransactionCheckListener checkListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionCheckListener(); + } + + return null; + } + + @Override + public TransactionListener getCheckListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionListener(); + } + return null; + } + + @Override + public void checkTransactionState(final String addr, final MessageExt msg, + final CheckTransactionStateRequestHeader header) { + Runnable request = new Runnable() { + private final String brokerAddr = addr; + private final MessageExt message = msg; + private final CheckTransactionStateRequestHeader checkRequestHeader = header; + private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); + + @Override + public void run() { + TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); + TransactionListener transactionListener = getCheckListener(); + if (transactionCheckListener != null || transactionListener != null) { + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable exception = null; + try { + if (transactionCheckListener != null) { + localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + } else if (transactionListener != null) { + log.debug("Used new check API in transaction message"); + localTransactionState = transactionListener.checkLocalTransaction(message); + } else { + log.warn("CheckTransactionState, pick transactionListener by group[{}] failed", group); + } + } catch (Throwable e) { + log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); + exception = e; + } + + this.processTransactionState( + localTransactionState, + group, + exception); + } else { + log.warn("CheckTransactionState, pick transactionCheckListener by group[{}] failed", group); + } + } + + private void processTransactionState( + final LocalTransactionState localTransactionState, + final String producerGroup, + final Throwable exception) { + final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); + thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); + thisHeader.setProducerGroup(producerGroup); + thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); + thisHeader.setFromTransactionCheck(true); + + String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (uniqueKey == null) { + uniqueKey = message.getMsgId(); + } + thisHeader.setMsgId(uniqueKey); + thisHeader.setTransactionId(checkRequestHeader.getTransactionId()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + log.warn("when broker check, client rollback this transaction, {}", thisHeader); + break; + case UNKNOW: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + log.warn("when broker check, client does not know this transaction state, {}", thisHeader); + break; + default: + break; + } + + String remark = null; + if (exception != null) { + remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); + } + + try { + DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, + 3000); + } catch (Exception e) { + log.error("endTransactionOneway exception", e); + } + } + }; + + this.checkExecutor.submit(request); + } + + @Override + public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { + if (info != null && topic != null) { + TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); + if (prev != null) { + log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); + } + } + } + + @Override + public boolean isUnitMode() { + return this.defaultMQProducer.isUnitMode(); + } + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.makeSureStateOK(); + Validators.checkTopic(newTopic); + + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The producer service state not OK, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + } + } + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + public MessageExt viewMessage( + String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.makeSureStateOK(); + + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); + } + + /** + * DEFAULT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, + SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws RejectedExecutionException + */ + @Deprecated + public void send(final Message msg, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException( + new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); + } + + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { + return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); + } + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); + } + + private void validateNameServerSetting() throws MQClientException { + List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); + if (null == nsList || nsList.isEmpty()) { + throw new MQClientException( + "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); + } + + } + + private SendResult sendDefaultImpl( + Message msg, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final long timeout + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + final long invokeID = random.nextLong(); + long beginTimestampFirst = System.currentTimeMillis(); + long beginTimestampPrev = beginTimestampFirst; + long endTimestamp = beginTimestampFirst; + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + boolean callTimeout = false; + MessageQueue mq = null; + Exception exception = null; + SendResult sendResult = null; + int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; + int times = 0; + String[] brokersSent = new String[timesTotal]; + for (; times < timesTotal; times++) { + String lastBrokerName = null == mq ? null : mq.getBrokerName(); + MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); + if (mqSelected != null) { + mq = mqSelected; + brokersSent[times] = mq.getBrokerName(); + try { + beginTimestampPrev = System.currentTimeMillis(); + if (times > 0) { + //Reset topic with namespace during resend. + msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic())); + } + long costTime = beginTimestampPrev - beginTimestampFirst; + if (timeout < costTime) { + callTimeout = true; + break; + } + + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + switch (communicationMode) { + case ASYNC: + return null; + case ONEWAY: + return null; + case SYNC: + if (sendResult.getSendStatus() != SendStatus.SEND_OK) { + if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { + continue; + } + } + + return sendResult; + default: + break; + } + } catch (RemotingException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQClientException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQBrokerException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + switch (e.getResponseCode()) { + case ResponseCode.TOPIC_NOT_EXIST: + case ResponseCode.SERVICE_NOT_AVAILABLE: + case ResponseCode.SYSTEM_ERROR: + case ResponseCode.NO_PERMISSION: + case ResponseCode.NO_BUYER_ID: + case ResponseCode.NOT_IN_CURRENT_UNIT: + continue; + default: + if (sendResult != null) { + return sendResult; + } + + throw e; + } + } catch (InterruptedException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + throw e; + } + } else { + break; + } + } + + if (sendResult != null) { + return sendResult; + } + + String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", + times, + System.currentTimeMillis() - beginTimestampFirst, + msg.getTopic(), + Arrays.toString(brokersSent)); + + info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); + + MQClientException mqClientException = new MQClientException(info, exception); + if (callTimeout) { + throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout"); + } + + if (exception instanceof MQBrokerException) { + mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); + } else if (exception instanceof RemotingConnectException) { + mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION); + } else if (exception instanceof RemotingTimeoutException) { + mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT); + } else if (exception instanceof MQClientException) { + mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION); + } + + throw mqClientException; + } + + validateNameServerSetting(); + + throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), + null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); + } + + private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { + TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); + if (null == topicPublishInfo || !topicPublishInfo.ok()) { + this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + } + + if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { + return topicPublishInfo; + } else { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + return topicPublishInfo; + } + } + + private SendResult sendKernelImpl(final Message msg, + final MessageQueue mq, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + tryToFindTopicPublishInfo(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + SendMessageContext context = null; + if (brokerAddr != null) { + brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr); + + byte[] prevBody = msg.getBody(); + try { + //for MessageBatch,ID has been set in the generating process + if (!(msg instanceof MessageBatch)) { + MessageClientIDSetter.setUniqID(msg); + } + + boolean topicWithNamespace = false; + if (null != this.mQClientFactory.getClientConfig().getNamespace()) { + msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace()); + topicWithNamespace = true; + } + + int sysFlag = 0; + boolean msgBodyCompressed = false; + if (this.tryToCompressMessage(msg)) { + sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + msgBodyCompressed = true; + } + + final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + } + + if (hasCheckForbiddenHook()) { + CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); + checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); + checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); + checkForbiddenContext.setCommunicationMode(communicationMode); + checkForbiddenContext.setBrokerAddr(brokerAddr); + checkForbiddenContext.setMessage(msg); + checkForbiddenContext.setMq(mq); + checkForbiddenContext.setUnitMode(this.isUnitMode()); + this.executeCheckForbiddenHook(checkForbiddenContext); + } + + if (this.hasSendMessageHook()) { + context = new SendMessageContext(); + context.setProducer(this); + context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + context.setCommunicationMode(communicationMode); + context.setBornHost(this.defaultMQProducer.getClientIP()); + context.setBrokerAddr(brokerAddr); + context.setMessage(msg); + context.setMq(mq); + context.setNamespace(this.defaultMQProducer.getNamespace()); + String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (isTrans != null && isTrans.equals("true")) { + context.setMsgType(MessageType.Trans_Msg_Half); + } + + if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) { + context.setMsgType(MessageType.Delay_Msg); + } + this.executeSendMessageHookBefore(context); + } + + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); + requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setSysFlag(sysFlag); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + requestHeader.setReconsumeTimes(0); + requestHeader.setUnitMode(this.isUnitMode()); + requestHeader.setBatch(msg instanceof MessageBatch); + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); + if (reconsumeTimes != null) { + requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); + } + + String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg); + if (maxReconsumeTimes != null) { + requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); + } + } + + SendResult sendResult = null; + switch (communicationMode) { + case ASYNC: + Message tmpMessage = msg; + boolean messageCloned = false; + if (msgBodyCompressed) { + //If msg body was compressed, msgbody should be reset using prevBody. + //Clone new message using commpressed message body and recover origin massage. + //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 + tmpMessage = MessageAccessor.cloneMessage(msg); + messageCloned = true; + msg.setBody(prevBody); + } + + if (topicWithNamespace) { + if (!messageCloned) { + tmpMessage = MessageAccessor.cloneMessage(msg); + messageCloned = true; + } + msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); + } + + long costTimeAsync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeAsync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( + brokerAddr, + mq.getBrokerName(), + tmpMessage, + requestHeader, + timeout - costTimeAsync, + communicationMode, + sendCallback, + topicPublishInfo, + this.mQClientFactory, + this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(), + context, + this); + break; + case ONEWAY: + case SYNC: + long costTimeSync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeSync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( + brokerAddr, + mq.getBrokerName(), + msg, + requestHeader, + timeout - costTimeSync, + communicationMode, + context, + this); + break; + default: + assert false; + break; + } + + if (this.hasSendMessageHook()) { + context.setSendResult(sendResult); + this.executeSendMessageHookAfter(context); + } + + return sendResult; + } catch (RemotingException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (MQBrokerException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (InterruptedException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } finally { + msg.setBody(prevBody); + msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + private boolean tryToCompressMessage(final Message msg) { + if (msg instanceof MessageBatch) { + //batch dose not support compressing right now + return false; + } + byte[] body = msg.getBody(); + if (body != null) { + if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { + try { + byte[] data = UtilAll.compress(body, zipCompressLevel); + if (data != null) { + msg.setBody(data); + return true; + } + } catch (IOException e) { + log.error("tryToCompressMessage exception", e); + log.warn(msg.toString()); + } + } + } + + return false; + } + + public boolean hasCheckForbiddenHook() { + return !checkForbiddenHookList.isEmpty(); + } + + public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { + if (hasCheckForbiddenHook()) { + for (CheckForbiddenHook hook : checkForbiddenHookList) { + hook.checkForbidden(context); + } + } + } + + public boolean hasSendMessageHook() { + return !this.sendMessageHookList.isEmpty(); + } + + public void executeSendMessageHookBefore(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageBefore(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookBefore", e); + } + } + } + } + + public void executeSendMessageHookAfter(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageAfter(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookAfter", e); + } + } + } + } + + /** + * DEFAULT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * KERNEL SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueue mq) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueue mq, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("call timeout"); + } + + return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); + } + + /** + * KERNEL ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param mq + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + try { + makeSureStateOK(); + Validators.checkMessage(msg, defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } catch (Exception e) { + sendCallback.onException(e); + } + + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); + } + + } + + /** + * KERNEL ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, + MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * SELECT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout); + } + + private SendResult sendSelectImpl( + Message msg, + MessageQueueSelector selector, + Object arg, + final CommunicationMode communicationMode, + final SendCallback sendCallback, final long timeout + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + try { + List messageQueueList = + mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); + Message userMessage = MessageAccessor.cloneMessage(msg); + String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); + userMessage.setTopic(userTopic); + + mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg)); + } catch (Throwable e) { + throw new MQClientException("select message queue throwed exception.", e); + } + + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("sendSelectImpl call timeout"); + } + if (mq != null) { + return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime); + } else { + throw new MQClientException("select message queue return null.", null); + } + } + + validateNameServerSetting(); + throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); + } + + /** + * SELECT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param selector + * @param arg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + try { + sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknownn exception", e); + } + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("exector rejected ", e); + } + } + + /** + * SELECT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter localTransactionExecuter, final Object arg) + throws MQClientException { + TransactionListener transactionListener = getCheckListener(); + if (null == localTransactionExecuter && null == transactionListener) { + throw new MQClientException("tranExecutor is null", null); + } + + // ignore DelayTimeLevel parameter + if (msg.getDelayTimeLevel() != 0) { + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + } + + Validators.checkMessage(msg, this.defaultMQProducer); + + SendResult sendResult = null; + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup()); + try { + sendResult = this.send(msg); + } catch (Exception e) { + throw new MQClientException("send message Exception", e); + } + + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable localException = null; + switch (sendResult.getSendStatus()) { + case SEND_OK: { + try { + if (sendResult.getTransactionId() != null) { + msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); + } + String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + msg.setTransactionId(transactionId); + } + if (null != localTransactionExecuter) { + localTransactionState = localTransactionExecuter.executeLocalTransactionBranch(msg, arg); + } else if (transactionListener != null) { + log.debug("Used new transaction API"); + localTransactionState = transactionListener.executeLocalTransaction(msg, arg); + } + if (null == localTransactionState) { + localTransactionState = LocalTransactionState.UNKNOW; + } + + if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { + log.info("executeLocalTransactionBranch return {}", localTransactionState); + log.info(msg.toString()); + } + } catch (Throwable e) { + log.info("executeLocalTransactionBranch exception", e); + log.info(msg.toString()); + localException = e; + } + } + break; + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; + break; + default: + break; + } + + try { + this.endTransaction(sendResult, localTransactionState, localException); + } catch (Exception e) { + log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); + } + + TransactionSendResult transactionSendResult = new TransactionSendResult(); + transactionSendResult.setSendStatus(sendResult.getSendStatus()); + transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); + transactionSendResult.setMsgId(sendResult.getMsgId()); + transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); + transactionSendResult.setTransactionId(sendResult.getTransactionId()); + transactionSendResult.setLocalTransactionState(localTransactionState); + return transactionSendResult; + } + + /** + * DEFAULT SYNC ------------------------------------------------------- + */ + public SendResult send( + Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void endTransaction( + final SendResult sendResult, + final LocalTransactionState localTransactionState, + final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { + final MessageId id; + if (sendResult.getOffsetMsgId() != null) { + id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); + } else { + id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); + } + String transactionId = sendResult.getTransactionId(); + final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); + EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); + requestHeader.setTransactionId(transactionId); + requestHeader.setCommitLogOffset(id.getOffset()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + break; + case UNKNOW: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + break; + default: + break; + } + + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); + requestHeader.setMsgId(sendResult.getMsgId()); + String remark = localException != null ? ("executeLocalTransactionBranch exception: " + localException.toString()) : null; + this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark, + this.defaultMQProducer.getSendMsgTimeout()); + } + + public void setCallbackExecutor(final ExecutorService callbackExecutor) { + this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); + } + + public ExecutorService getAsyncSenderExecutor() { + return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor; + } + + public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) { + this.asyncSenderExecutor = asyncSenderExecutor; + } + + public SendResult send(Message msg, + long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); + } + + public ConcurrentMap getTopicPublishInfoTable() { + return topicPublishInfoTable; + } + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + public ServiceState getServiceState() { + return serviceState; + } + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + public long[] getNotAvailableDuration() { + return this.mqFaultStrategy.getNotAvailableDuration(); + } + + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration); + } + + public long[] getLatencyMax() { + return this.mqFaultStrategy.getLatencyMax(); + } + + public void setLatencyMax(final long[] latencyMax) { + this.mqFaultStrategy.setLatencyMax(latencyMax); + } + + public boolean isSendLatencyFaultEnable() { + return this.mqFaultStrategy.isSendLatencyFaultEnable(); + } + + public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { + this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable); + } + + public DefaultMQProducer getDefaultMQProducer() { + return defaultMQProducer; + } } From 8488207cd906fcac596ef52988fdae11adfc6a86 Mon Sep 17 00:00:00 2001 From: huangying Date: Fri, 18 Oct 2019 14:38:31 +0800 Subject: [PATCH 40/73] [#1540] format checkstyle --- .../impl/producer/DefaultMQProducerImpl.java | 2569 ++++++++--------- 1 file changed, 1284 insertions(+), 1285 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java index 8346dc7ed2..5b7ba9c6f7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -34,7 +34,6 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.common.ClientErrorCode; @@ -89,1288 +88,1288 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; public class DefaultMQProducerImpl implements MQProducerInner { - private final InternalLogger log = ClientLogger.getLog(); - private final Random random = new Random(); - private final DefaultMQProducer defaultMQProducer; - private final ConcurrentMap topicPublishInfoTable = - new ConcurrentHashMap(); - private final ArrayList sendMessageHookList = new ArrayList(); - private final RPCHook rpcHook; - protected BlockingQueue checkRequestQueue; - protected ExecutorService checkExecutor; - private ServiceState serviceState = ServiceState.CREATE_JUST; - private MQClientInstance mQClientFactory; - private ArrayList checkForbiddenHookList = new ArrayList(); - private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); - - private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); - - private final BlockingQueue asyncSenderThreadPoolQueue; - private final ExecutorService defaultAsyncSenderExecutor; - private ExecutorService asyncSenderExecutor; - - public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { - this(defaultMQProducer, null); - } - - public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { - this.defaultMQProducer = defaultMQProducer; - this.rpcHook = rpcHook; - - this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue(50000); - this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( - Runtime.getRuntime().availableProcessors(), - Runtime.getRuntime().availableProcessors(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.asyncSenderThreadPoolQueue, - new ThreadFactory() { - private AtomicInteger threadIndex = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet()); - } - }); - } - - public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { - this.checkForbiddenHookList.add(checkForbiddenHook); - log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), - checkForbiddenHookList.size()); - } - - public void initTransactionEnv() { - TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; - if (producer.getExecutorService() != null) { - this.checkExecutor = producer.getExecutorService(); - } else { - this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); - this.checkExecutor = new ThreadPoolExecutor( - producer.getCheckThreadPoolMinSize(), - producer.getCheckThreadPoolMaxSize(), - 1000 * 60, - TimeUnit.MILLISECONDS, - this.checkRequestQueue); - } - } - - public void destroyTransactionEnv() { - if (this.checkExecutor != null) { - this.checkExecutor.shutdown(); - } - } - - public void registerSendMessageHook(final SendMessageHook hook) { - this.sendMessageHookList.add(hook); - log.info("register sendMessage Hook, {}", hook.hookName()); - } - - public void start() throws MQClientException { - this.start(true); - } - - public void start(final boolean startFactory) throws MQClientException { - switch (this.serviceState) { - case CREATE_JUST: - this.serviceState = ServiceState.START_FAILED; - - this.checkConfig(); - - if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { - this.defaultMQProducer.changeInstanceNameToPID(); - } - - this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); - - boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); - if (!registerOK) { - this.serviceState = ServiceState.CREATE_JUST; - throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() - + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), - null); - } - - this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); - - if (startFactory) { - mQClientFactory.start(); - } - - log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), - this.defaultMQProducer.isSendMessageWithVIPChannel()); - this.serviceState = ServiceState.RUNNING; - break; - case RUNNING: - case START_FAILED: - case SHUTDOWN_ALREADY: - throw new MQClientException("The producer service state not OK, maybe started once, " - + this.serviceState - + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), - null); - default: - break; - } - - this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); - } - - private void checkConfig() throws MQClientException { - Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); - - if (null == this.defaultMQProducer.getProducerGroup()) { - throw new MQClientException("producerGroup is null", null); - } - - if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { - throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", - null); - } - } - - public void shutdown() { - this.shutdown(true); - } - - public void shutdown(final boolean shutdownFactory) { - switch (this.serviceState) { - case CREATE_JUST: - break; - case RUNNING: - this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); - this.defaultAsyncSenderExecutor.shutdown(); - if (shutdownFactory) { - this.mQClientFactory.shutdown(); - } - - log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); - this.serviceState = ServiceState.SHUTDOWN_ALREADY; - break; - case SHUTDOWN_ALREADY: - break; - default: - break; - } - } - - @Override - public Set getPublishTopicList() { - Set topicList = new HashSet(); - for (String key : this.topicPublishInfoTable.keySet()) { - topicList.add(key); - } - - return topicList; - } - - @Override - public boolean isPublishTopicNeedUpdate(String topic) { - TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); - - return null == prev || !prev.ok(); - } - - /** - * This method will be removed in the version 5.0.0 and getCheckListener is recommended. - * - * @return - */ - @Override - @Deprecated - public TransactionCheckListener checkListener() { - if (this.defaultMQProducer instanceof TransactionMQProducer) { - TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionCheckListener(); - } - - return null; - } - - @Override - public TransactionListener getCheckListener() { - if (this.defaultMQProducer instanceof TransactionMQProducer) { - TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; - return producer.getTransactionListener(); - } - return null; - } - - @Override - public void checkTransactionState(final String addr, final MessageExt msg, - final CheckTransactionStateRequestHeader header) { - Runnable request = new Runnable() { - private final String brokerAddr = addr; - private final MessageExt message = msg; - private final CheckTransactionStateRequestHeader checkRequestHeader = header; - private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); - - @Override - public void run() { - TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); - TransactionListener transactionListener = getCheckListener(); - if (transactionCheckListener != null || transactionListener != null) { - LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; - Throwable exception = null; - try { - if (transactionCheckListener != null) { - localTransactionState = transactionCheckListener.checkLocalTransactionState(message); - } else if (transactionListener != null) { - log.debug("Used new check API in transaction message"); - localTransactionState = transactionListener.checkLocalTransaction(message); - } else { - log.warn("CheckTransactionState, pick transactionListener by group[{}] failed", group); - } - } catch (Throwable e) { - log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); - exception = e; - } - - this.processTransactionState( - localTransactionState, - group, - exception); - } else { - log.warn("CheckTransactionState, pick transactionCheckListener by group[{}] failed", group); - } - } - - private void processTransactionState( - final LocalTransactionState localTransactionState, - final String producerGroup, - final Throwable exception) { - final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); - thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); - thisHeader.setProducerGroup(producerGroup); - thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); - thisHeader.setFromTransactionCheck(true); - - String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); - if (uniqueKey == null) { - uniqueKey = message.getMsgId(); - } - thisHeader.setMsgId(uniqueKey); - thisHeader.setTransactionId(checkRequestHeader.getTransactionId()); - switch (localTransactionState) { - case COMMIT_MESSAGE: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); - break; - case ROLLBACK_MESSAGE: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); - log.warn("when broker check, client rollback this transaction, {}", thisHeader); - break; - case UNKNOW: - thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); - log.warn("when broker check, client does not know this transaction state, {}", thisHeader); - break; - default: - break; - } - - String remark = null; - if (exception != null) { - remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); - } - - try { - DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, - 3000); - } catch (Exception e) { - log.error("endTransactionOneway exception", e); - } - } - }; - - this.checkExecutor.submit(request); - } - - @Override - public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { - if (info != null && topic != null) { - TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); - if (prev != null) { - log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); - } - } - } - - @Override - public boolean isUnitMode() { - return this.defaultMQProducer.isUnitMode(); - } - - public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { - createTopic(key, newTopic, queueNum, 0); - } - - public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { - this.makeSureStateOK(); - Validators.checkTopic(newTopic); - - this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); - } - - private void makeSureStateOK() throws MQClientException { - if (this.serviceState != ServiceState.RUNNING) { - throw new MQClientException("The producer service state not OK, " - + this.serviceState - + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), - null); - } - } - - public List fetchPublishMessageQueues(String topic) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); - } - - public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); - } - - public long maxOffset(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); - } - - public long minOffset(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().minOffset(mq); - } - - public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); - } - - public MessageExt viewMessage( - String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - this.makeSureStateOK(); - - return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); - } - - public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) - throws MQClientException, InterruptedException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); - } - - public MessageExt queryMessageByUniqKey(String topic, String uniqKey) - throws MQClientException, InterruptedException { - this.makeSureStateOK(); - return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); - } - - /** - * DEFAULT ASYNC ------------------------------------------------------- - */ - public void send(Message msg, - SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { - send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws RejectedExecutionException - */ - @Deprecated - public void send(final Message msg, final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); - } catch (Exception e) { - sendCallback.onException(e); - } - } else { - sendCallback.onException( - new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); - } - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("executor rejected ", e); - } - - } - - public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { - return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); - } - - public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { - this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); - } - - private void validateNameServerSetting() throws MQClientException { - List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); - if (null == nsList || nsList.isEmpty()) { - throw new MQClientException( - "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); - } - - } - - private SendResult sendDefaultImpl( - Message msg, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final long timeout - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - final long invokeID = random.nextLong(); - long beginTimestampFirst = System.currentTimeMillis(); - long beginTimestampPrev = beginTimestampFirst; - long endTimestamp = beginTimestampFirst; - TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); - if (topicPublishInfo != null && topicPublishInfo.ok()) { - boolean callTimeout = false; - MessageQueue mq = null; - Exception exception = null; - SendResult sendResult = null; - int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; - int times = 0; - String[] brokersSent = new String[timesTotal]; - for (; times < timesTotal; times++) { - String lastBrokerName = null == mq ? null : mq.getBrokerName(); - MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); - if (mqSelected != null) { - mq = mqSelected; - brokersSent[times] = mq.getBrokerName(); - try { - beginTimestampPrev = System.currentTimeMillis(); - if (times > 0) { - //Reset topic with namespace during resend. - msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic())); - } - long costTime = beginTimestampPrev - beginTimestampFirst; - if (timeout < costTime) { - callTimeout = true; - break; - } - - sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); - switch (communicationMode) { - case ASYNC: - return null; - case ONEWAY: - return null; - case SYNC: - if (sendResult.getSendStatus() != SendStatus.SEND_OK) { - if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { - continue; - } - } - - return sendResult; - default: - break; - } - } catch (RemotingException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - continue; - } catch (MQClientException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - continue; - } catch (MQBrokerException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); - log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - exception = e; - switch (e.getResponseCode()) { - case ResponseCode.TOPIC_NOT_EXIST: - case ResponseCode.SERVICE_NOT_AVAILABLE: - case ResponseCode.SYSTEM_ERROR: - case ResponseCode.NO_PERMISSION: - case ResponseCode.NO_BUYER_ID: - case ResponseCode.NOT_IN_CURRENT_UNIT: - continue; - default: - if (sendResult != null) { - return sendResult; - } - - throw e; - } - } catch (InterruptedException e) { - endTimestamp = System.currentTimeMillis(); - this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); - log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); - log.warn(msg.toString()); - - log.warn("sendKernelImpl exception", e); - log.warn(msg.toString()); - throw e; - } - } else { - break; - } - } - - if (sendResult != null) { - return sendResult; - } - - String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", - times, - System.currentTimeMillis() - beginTimestampFirst, - msg.getTopic(), - Arrays.toString(brokersSent)); - - info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); - - MQClientException mqClientException = new MQClientException(info, exception); - if (callTimeout) { - throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout"); - } - - if (exception instanceof MQBrokerException) { - mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); - } else if (exception instanceof RemotingConnectException) { - mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION); - } else if (exception instanceof RemotingTimeoutException) { - mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT); - } else if (exception instanceof MQClientException) { - mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION); - } - - throw mqClientException; - } - - validateNameServerSetting(); - - throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), - null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); - } - - private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { - TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); - if (null == topicPublishInfo || !topicPublishInfo.ok()) { - this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - topicPublishInfo = this.topicPublishInfoTable.get(topic); - } - - if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { - return topicPublishInfo; - } else { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); - topicPublishInfo = this.topicPublishInfoTable.get(topic); - return topicPublishInfo; - } - } - - private SendResult sendKernelImpl(final Message msg, - final MessageQueue mq, - final CommunicationMode communicationMode, - final SendCallback sendCallback, - final TopicPublishInfo topicPublishInfo, - final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - if (null == brokerAddr) { - tryToFindTopicPublishInfo(mq.getTopic()); - brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); - } - - SendMessageContext context = null; - if (brokerAddr != null) { - brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr); - - byte[] prevBody = msg.getBody(); - try { - //for MessageBatch,ID has been set in the generating process - if (!(msg instanceof MessageBatch)) { - MessageClientIDSetter.setUniqID(msg); - } - - boolean topicWithNamespace = false; - if (null != this.mQClientFactory.getClientConfig().getNamespace()) { - msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace()); - topicWithNamespace = true; - } - - int sysFlag = 0; - boolean msgBodyCompressed = false; - if (this.tryToCompressMessage(msg)) { - sysFlag |= MessageSysFlag.COMPRESSED_FLAG; - msgBodyCompressed = true; - } - - final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { - sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; - } - - if (hasCheckForbiddenHook()) { - CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); - checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); - checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); - checkForbiddenContext.setCommunicationMode(communicationMode); - checkForbiddenContext.setBrokerAddr(brokerAddr); - checkForbiddenContext.setMessage(msg); - checkForbiddenContext.setMq(mq); - checkForbiddenContext.setUnitMode(this.isUnitMode()); - this.executeCheckForbiddenHook(checkForbiddenContext); - } - - if (this.hasSendMessageHook()) { - context = new SendMessageContext(); - context.setProducer(this); - context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - context.setCommunicationMode(communicationMode); - context.setBornHost(this.defaultMQProducer.getClientIP()); - context.setBrokerAddr(brokerAddr); - context.setMessage(msg); - context.setMq(mq); - context.setNamespace(this.defaultMQProducer.getNamespace()); - String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); - if (isTrans != null && isTrans.equals("true")) { - context.setMsgType(MessageType.Trans_Msg_Half); - } - - if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) { - context.setMsgType(MessageType.Delay_Msg); - } - this.executeSendMessageHookBefore(context); - } - - SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); - requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - requestHeader.setTopic(msg.getTopic()); - requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); - requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); - requestHeader.setQueueId(mq.getQueueId()); - requestHeader.setSysFlag(sysFlag); - requestHeader.setBornTimestamp(System.currentTimeMillis()); - requestHeader.setFlag(msg.getFlag()); - requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); - requestHeader.setReconsumeTimes(0); - requestHeader.setUnitMode(this.isUnitMode()); - requestHeader.setBatch(msg instanceof MessageBatch); - if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { - String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); - if (reconsumeTimes != null) { - requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); - } - - String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg); - if (maxReconsumeTimes != null) { - requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); - } - } - - SendResult sendResult = null; - switch (communicationMode) { - case ASYNC: - Message tmpMessage = msg; - boolean messageCloned = false; - if (msgBodyCompressed) { - //If msg body was compressed, msgbody should be reset using prevBody. - //Clone new message using commpressed message body and recover origin massage. - //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 - tmpMessage = MessageAccessor.cloneMessage(msg); - messageCloned = true; - msg.setBody(prevBody); - } - - if (topicWithNamespace) { - if (!messageCloned) { - tmpMessage = MessageAccessor.cloneMessage(msg); - messageCloned = true; - } - msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); - } - - long costTimeAsync = System.currentTimeMillis() - beginStartTime; - if (timeout < costTimeAsync) { - throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); - } - sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( - brokerAddr, - mq.getBrokerName(), - tmpMessage, - requestHeader, - timeout - costTimeAsync, - communicationMode, - sendCallback, - topicPublishInfo, - this.mQClientFactory, - this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(), - context, - this); - break; - case ONEWAY: - case SYNC: - long costTimeSync = System.currentTimeMillis() - beginStartTime; - if (timeout < costTimeSync) { - throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); - } - sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( - brokerAddr, - mq.getBrokerName(), - msg, - requestHeader, - timeout - costTimeSync, - communicationMode, - context, - this); - break; - default: - assert false; - break; - } - - if (this.hasSendMessageHook()) { - context.setSendResult(sendResult); - this.executeSendMessageHookAfter(context); - } - - return sendResult; - } catch (RemotingException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } catch (MQBrokerException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } catch (InterruptedException e) { - if (this.hasSendMessageHook()) { - context.setException(e); - this.executeSendMessageHookAfter(context); - } - throw e; - } finally { - msg.setBody(prevBody); - msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); - } - } - - throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); - } - - public MQClientInstance getmQClientFactory() { - return mQClientFactory; - } - - private boolean tryToCompressMessage(final Message msg) { - if (msg instanceof MessageBatch) { - //batch dose not support compressing right now - return false; - } - byte[] body = msg.getBody(); - if (body != null) { - if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { - try { - byte[] data = UtilAll.compress(body, zipCompressLevel); - if (data != null) { - msg.setBody(data); - return true; - } - } catch (IOException e) { - log.error("tryToCompressMessage exception", e); - log.warn(msg.toString()); - } - } - } - - return false; - } - - public boolean hasCheckForbiddenHook() { - return !checkForbiddenHookList.isEmpty(); - } - - public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { - if (hasCheckForbiddenHook()) { - for (CheckForbiddenHook hook : checkForbiddenHookList) { - hook.checkForbidden(context); - } - } - } - - public boolean hasSendMessageHook() { - return !this.sendMessageHookList.isEmpty(); - } - - public void executeSendMessageHookBefore(final SendMessageContext context) { - if (!this.sendMessageHookList.isEmpty()) { - for (SendMessageHook hook : this.sendMessageHookList) { - try { - hook.sendMessageBefore(context); - } catch (Throwable e) { - log.warn("failed to executeSendMessageHookBefore", e); - } - } - } - } - - public void executeSendMessageHookAfter(final SendMessageContext context) { - if (!this.sendMessageHookList.isEmpty()) { - for (SendMessageHook hook : this.sendMessageHookList) { - try { - hook.sendMessageAfter(context); - } catch (Throwable e) { - log.warn("failed to executeSendMessageHookAfter", e); - } - } - } - } - - /** - * DEFAULT ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { - try { - this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - /** - * KERNEL SYNC ------------------------------------------------------- - */ - public SendResult send(Message msg, MessageQueue mq) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout()); - } - - public SendResult send(Message msg, MessageQueue mq, long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - if (!msg.getTopic().equals(mq.getTopic())) { - throw new MQClientException("message's topic not equal mq's topic", null); - } - - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout < costTime) { - throw new RemotingTooMuchRequestException("call timeout"); - } - - return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); - } - - /** - * KERNEL ASYNC ------------------------------------------------------- - */ - public void send(Message msg, MessageQueue mq, SendCallback sendCallback) - throws MQClientException, RemotingException, InterruptedException { - send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param mq - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException - */ - @Deprecated - public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - try { - makeSureStateOK(); - Validators.checkMessage(msg, defaultMQProducer); - - if (!msg.getTopic().equals(mq.getTopic())) { - throw new MQClientException("message's topic not equal mq's topic", null); - } - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, - timeout - costTime); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); - } - } catch (Exception e) { - sendCallback.onException(e); - } - - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("executor rejected ", e); - } - - } - - /** - * KERNEL ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg, - MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - try { - this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - /** - * SELECT SYNC ------------------------------------------------------- - */ - public SendResult send(Message msg, MessageQueueSelector selector, Object arg) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout()); - } - - public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout); - } - - private SendResult sendSelectImpl( - Message msg, - MessageQueueSelector selector, - Object arg, - final CommunicationMode communicationMode, - final SendCallback sendCallback, final long timeout - ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - long beginStartTime = System.currentTimeMillis(); - this.makeSureStateOK(); - Validators.checkMessage(msg, this.defaultMQProducer); - - TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); - if (topicPublishInfo != null && topicPublishInfo.ok()) { - MessageQueue mq = null; - try { - List messageQueueList = - mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); - Message userMessage = MessageAccessor.cloneMessage(msg); - String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); - userMessage.setTopic(userTopic); - - mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg)); - } catch (Throwable e) { - throw new MQClientException("select message queue throwed exception.", e); - } - - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout < costTime) { - throw new RemotingTooMuchRequestException("sendSelectImpl call timeout"); - } - if (mq != null) { - return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime); - } else { - throw new MQClientException("select message queue return null.", null); - } - } - - validateNameServerSetting(); - throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); - } - - /** - * SELECT ASYNC ------------------------------------------------------- - */ - public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) - throws MQClientException, RemotingException, InterruptedException { - send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); - } - - /** - * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be - * provided in next version - * - * @param msg - * @param selector - * @param arg - * @param sendCallback - * @param timeout the sendCallback will be invoked at most time - * @throws MQClientException - * @throws RemotingException - * @throws InterruptedException - */ - @Deprecated - public void send(final Message msg, final MessageQueueSelector selector, final Object arg, - final SendCallback sendCallback, final long timeout) - throws MQClientException, RemotingException, InterruptedException { - final long beginStartTime = System.currentTimeMillis(); - ExecutorService executor = this.getAsyncSenderExecutor(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - long costTime = System.currentTimeMillis() - beginStartTime; - if (timeout > costTime) { - try { - try { - sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, - timeout - costTime); - } catch (MQBrokerException e) { - throw new MQClientException("unknownn exception", e); - } - } catch (Exception e) { - sendCallback.onException(e); - } - } else { - sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); - } - } - - }); - } catch (RejectedExecutionException e) { - throw new MQClientException("exector rejected ", e); - } - } - - /** - * SELECT ONEWAY ------------------------------------------------------- - */ - public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) - throws MQClientException, RemotingException, InterruptedException { - try { - this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); - } catch (MQBrokerException e) { - throw new MQClientException("unknown exception", e); - } - } - - public TransactionSendResult sendMessageInTransaction(final Message msg, - final LocalTransactionExecuter localTransactionExecuter, final Object arg) - throws MQClientException { - TransactionListener transactionListener = getCheckListener(); - if (null == localTransactionExecuter && null == transactionListener) { - throw new MQClientException("tranExecutor is null", null); - } - - // ignore DelayTimeLevel parameter - if (msg.getDelayTimeLevel() != 0) { - MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); - } - - Validators.checkMessage(msg, this.defaultMQProducer); - - SendResult sendResult = null; - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup()); - try { - sendResult = this.send(msg); - } catch (Exception e) { - throw new MQClientException("send message Exception", e); - } - - LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; - Throwable localException = null; - switch (sendResult.getSendStatus()) { - case SEND_OK: { - try { - if (sendResult.getTransactionId() != null) { - msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); - } - String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); - if (null != transactionId && !"".equals(transactionId)) { - msg.setTransactionId(transactionId); - } - if (null != localTransactionExecuter) { - localTransactionState = localTransactionExecuter.executeLocalTransactionBranch(msg, arg); - } else if (transactionListener != null) { - log.debug("Used new transaction API"); - localTransactionState = transactionListener.executeLocalTransaction(msg, arg); - } - if (null == localTransactionState) { - localTransactionState = LocalTransactionState.UNKNOW; - } - - if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { - log.info("executeLocalTransactionBranch return {}", localTransactionState); - log.info(msg.toString()); - } - } catch (Throwable e) { - log.info("executeLocalTransactionBranch exception", e); - log.info(msg.toString()); - localException = e; - } - } - break; - case FLUSH_DISK_TIMEOUT: - case FLUSH_SLAVE_TIMEOUT: - case SLAVE_NOT_AVAILABLE: - localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; - break; - default: - break; - } - - try { - this.endTransaction(sendResult, localTransactionState, localException); - } catch (Exception e) { - log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); - } - - TransactionSendResult transactionSendResult = new TransactionSendResult(); - transactionSendResult.setSendStatus(sendResult.getSendStatus()); - transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); - transactionSendResult.setMsgId(sendResult.getMsgId()); - transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); - transactionSendResult.setTransactionId(sendResult.getTransactionId()); - transactionSendResult.setLocalTransactionState(localTransactionState); - return transactionSendResult; - } - - /** - * DEFAULT SYNC ------------------------------------------------------- - */ - public SendResult send( - Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return send(msg, this.defaultMQProducer.getSendMsgTimeout()); - } - - public void endTransaction( - final SendResult sendResult, - final LocalTransactionState localTransactionState, - final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { - final MessageId id; - if (sendResult.getOffsetMsgId() != null) { - id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); - } else { - id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); - } - String transactionId = sendResult.getTransactionId(); - final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); - EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); - requestHeader.setTransactionId(transactionId); - requestHeader.setCommitLogOffset(id.getOffset()); - switch (localTransactionState) { - case COMMIT_MESSAGE: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); - break; - case ROLLBACK_MESSAGE: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); - break; - case UNKNOW: - requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); - break; - default: - break; - } - - requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); - requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); - requestHeader.setMsgId(sendResult.getMsgId()); - String remark = localException != null ? ("executeLocalTransactionBranch exception: " + localException.toString()) : null; - this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark, - this.defaultMQProducer.getSendMsgTimeout()); - } - - public void setCallbackExecutor(final ExecutorService callbackExecutor) { - this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); - } - - public ExecutorService getAsyncSenderExecutor() { - return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor; - } - - public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) { - this.asyncSenderExecutor = asyncSenderExecutor; - } - - public SendResult send(Message msg, - long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); - } - - public ConcurrentMap getTopicPublishInfoTable() { - return topicPublishInfoTable; - } - - public int getZipCompressLevel() { - return zipCompressLevel; - } - - public void setZipCompressLevel(int zipCompressLevel) { - this.zipCompressLevel = zipCompressLevel; - } - - public ServiceState getServiceState() { - return serviceState; - } - - public void setServiceState(ServiceState serviceState) { - this.serviceState = serviceState; - } - - public long[] getNotAvailableDuration() { - return this.mqFaultStrategy.getNotAvailableDuration(); - } - - public void setNotAvailableDuration(final long[] notAvailableDuration) { - this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration); - } - - public long[] getLatencyMax() { - return this.mqFaultStrategy.getLatencyMax(); - } - - public void setLatencyMax(final long[] latencyMax) { - this.mqFaultStrategy.setLatencyMax(latencyMax); - } - - public boolean isSendLatencyFaultEnable() { - return this.mqFaultStrategy.isSendLatencyFaultEnable(); - } - - public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { - this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable); - } - - public DefaultMQProducer getDefaultMQProducer() { - return defaultMQProducer; - } + private final InternalLogger log = ClientLogger.getLog(); + private final Random random = new Random(); + private final DefaultMQProducer defaultMQProducer; + private final ConcurrentMap topicPublishInfoTable = + new ConcurrentHashMap(); + private final ArrayList sendMessageHookList = new ArrayList(); + private final RPCHook rpcHook; + protected BlockingQueue checkRequestQueue; + protected ExecutorService checkExecutor; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private ArrayList checkForbiddenHookList = new ArrayList(); + private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); + + private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); + + private final BlockingQueue asyncSenderThreadPoolQueue; + private final ExecutorService defaultAsyncSenderExecutor; + private ExecutorService asyncSenderExecutor; + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { + this(defaultMQProducer, null); + } + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { + this.defaultMQProducer = defaultMQProducer; + this.rpcHook = rpcHook; + + this.asyncSenderThreadPoolQueue = new LinkedBlockingQueue(50000); + this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( + Runtime.getRuntime().availableProcessors(), + Runtime.getRuntime().availableProcessors(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.asyncSenderThreadPoolQueue, + new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "AsyncSenderExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + } + + public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { + this.checkForbiddenHookList.add(checkForbiddenHook); + log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), + checkForbiddenHookList.size()); + } + + public void initTransactionEnv() { + TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; + if (producer.getExecutorService() != null) { + this.checkExecutor = producer.getExecutorService(); + } else { + this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); + this.checkExecutor = new ThreadPoolExecutor( + producer.getCheckThreadPoolMinSize(), + producer.getCheckThreadPoolMaxSize(), + 1000 * 60, + TimeUnit.MILLISECONDS, + this.checkRequestQueue); + } + } + + public void destroyTransactionEnv() { + if (this.checkExecutor != null) { + this.checkExecutor.shutdown(); + } + } + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register sendMessage Hook, {}", hook.hookName()); + } + + public void start() throws MQClientException { + this.start(true); + } + + public void start(final boolean startFactory) throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { + this.defaultMQProducer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getOrCreateMQClientInstance(this.defaultMQProducer, rpcHook); + + boolean registerOK = mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + this.topicPublishInfoTable.put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); + + if (startFactory) { + mQClientFactory.start(); + } + + log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), + this.defaultMQProducer.isSendMessageWithVIPChannel()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The producer service state not OK, maybe started once, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + + private void checkConfig() throws MQClientException { + Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); + + if (null == this.defaultMQProducer.getProducerGroup()) { + throw new MQClientException("producerGroup is null", null); + } + + if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { + throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + ", please specify another one.", + null); + } + } + + public void shutdown() { + this.shutdown(true); + } + + public void shutdown(final boolean shutdownFactory) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); + this.defaultAsyncSenderExecutor.shutdown(); + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } + + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + @Override + public Set getPublishTopicList() { + Set topicList = new HashSet(); + for (String key : this.topicPublishInfoTable.keySet()) { + topicList.add(key); + } + + return topicList; + } + + @Override + public boolean isPublishTopicNeedUpdate(String topic) { + TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); + + return null == prev || !prev.ok(); + } + + /** + * This method will be removed in the version 5.0.0 and getCheckListener is recommended. + * + * @return + */ + @Override + @Deprecated + public TransactionCheckListener checkListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionCheckListener(); + } + + return null; + } + + @Override + public TransactionListener getCheckListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionListener(); + } + return null; + } + + @Override + public void checkTransactionState(final String addr, final MessageExt msg, + final CheckTransactionStateRequestHeader header) { + Runnable request = new Runnable() { + private final String brokerAddr = addr; + private final MessageExt message = msg; + private final CheckTransactionStateRequestHeader checkRequestHeader = header; + private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); + + @Override + public void run() { + TransactionCheckListener transactionCheckListener = DefaultMQProducerImpl.this.checkListener(); + TransactionListener transactionListener = getCheckListener(); + if (transactionCheckListener != null || transactionListener != null) { + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable exception = null; + try { + if (transactionCheckListener != null) { + localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + } else if (transactionListener != null) { + log.debug("Used new check API in transaction message"); + localTransactionState = transactionListener.checkLocalTransaction(message); + } else { + log.warn("CheckTransactionState, pick transactionListener by group[{}] failed", group); + } + } catch (Throwable e) { + log.error("Broker call checkTransactionState, but checkLocalTransactionState exception", e); + exception = e; + } + + this.processTransactionState( + localTransactionState, + group, + exception); + } else { + log.warn("CheckTransactionState, pick transactionCheckListener by group[{}] failed", group); + } + } + + private void processTransactionState( + final LocalTransactionState localTransactionState, + final String producerGroup, + final Throwable exception) { + final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); + thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); + thisHeader.setProducerGroup(producerGroup); + thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); + thisHeader.setFromTransactionCheck(true); + + String uniqueKey = message.getProperties().get(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (uniqueKey == null) { + uniqueKey = message.getMsgId(); + } + thisHeader.setMsgId(uniqueKey); + thisHeader.setTransactionId(checkRequestHeader.getTransactionId()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + log.warn("when broker check, client rollback this transaction, {}", thisHeader); + break; + case UNKNOW: + thisHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + log.warn("when broker check, client does not know this transaction state, {}", thisHeader); + break; + default: + break; + } + + String remark = null; + if (exception != null) { + remark = "checkLocalTransactionState Exception: " + RemotingHelper.exceptionSimpleDesc(exception); + } + + try { + DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, thisHeader, remark, + 3000); + } catch (Exception e) { + log.error("endTransactionOneway exception", e); + } + } + }; + + this.checkExecutor.submit(request); + } + + @Override + public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { + if (info != null && topic != null) { + TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); + if (prev != null) { + log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); + } + } + } + + @Override + public boolean isUnitMode() { + return this.defaultMQProducer.isUnitMode(); + } + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { + this.makeSureStateOK(); + Validators.checkTopic(newTopic); + + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The producer service state not OK, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + } + } + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + public MessageExt viewMessage( + String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.makeSureStateOK(); + + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + public MessageExt queryMessageByUniqKey(String topic, String uniqKey) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); + } + + /** + * DEFAULT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, + SendCallback sendCallback) throws MQClientException, RemotingException, InterruptedException { + send(msg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws RejectedExecutionException + */ + @Deprecated + public void send(final Message msg, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException( + new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); + } + + } + + public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { + return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); + } + + public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { + this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); + } + + private void validateNameServerSetting() throws MQClientException { + List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); + if (null == nsList || nsList.isEmpty()) { + throw new MQClientException( + "No name server address, please set it." + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null).setResponseCode(ClientErrorCode.NO_NAME_SERVER_EXCEPTION); + } + + } + + private SendResult sendDefaultImpl( + Message msg, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final long timeout + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + final long invokeID = random.nextLong(); + long beginTimestampFirst = System.currentTimeMillis(); + long beginTimestampPrev = beginTimestampFirst; + long endTimestamp = beginTimestampFirst; + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + boolean callTimeout = false; + MessageQueue mq = null; + Exception exception = null; + SendResult sendResult = null; + int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; + int times = 0; + String[] brokersSent = new String[timesTotal]; + for (; times < timesTotal; times++) { + String lastBrokerName = null == mq ? null : mq.getBrokerName(); + MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); + if (mqSelected != null) { + mq = mqSelected; + brokersSent[times] = mq.getBrokerName(); + try { + beginTimestampPrev = System.currentTimeMillis(); + if (times > 0) { + //Reset topic with namespace during resend. + msg.setTopic(this.defaultMQProducer.withNamespace(msg.getTopic())); + } + long costTime = beginTimestampPrev - beginTimestampFirst; + if (timeout < costTime) { + callTimeout = true; + break; + } + + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + switch (communicationMode) { + case ASYNC: + return null; + case ONEWAY: + return null; + case SYNC: + if (sendResult.getSendStatus() != SendStatus.SEND_OK) { + if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { + continue; + } + } + + return sendResult; + default: + break; + } + } catch (RemotingException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQClientException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + continue; + } catch (MQBrokerException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); + log.warn(String.format("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + exception = e; + switch (e.getResponseCode()) { + case ResponseCode.TOPIC_NOT_EXIST: + case ResponseCode.SERVICE_NOT_AVAILABLE: + case ResponseCode.SYSTEM_ERROR: + case ResponseCode.NO_PERMISSION: + case ResponseCode.NO_BUYER_ID: + case ResponseCode.NOT_IN_CURRENT_UNIT: + continue; + default: + if (sendResult != null) { + return sendResult; + } + + throw e; + } + } catch (InterruptedException e) { + endTimestamp = System.currentTimeMillis(); + this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); + log.warn(String.format("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq), e); + log.warn(msg.toString()); + + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + throw e; + } + } else { + break; + } + } + + if (sendResult != null) { + return sendResult; + } + + String info = String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", + times, + System.currentTimeMillis() - beginTimestampFirst, + msg.getTopic(), + Arrays.toString(brokersSent)); + + info += FAQUrl.suggestTodo(FAQUrl.SEND_MSG_FAILED); + + MQClientException mqClientException = new MQClientException(info, exception); + if (callTimeout) { + throw new RemotingTooMuchRequestException("sendDefaultImpl call timeout"); + } + + if (exception instanceof MQBrokerException) { + mqClientException.setResponseCode(((MQBrokerException) exception).getResponseCode()); + } else if (exception instanceof RemotingConnectException) { + mqClientException.setResponseCode(ClientErrorCode.CONNECT_BROKER_EXCEPTION); + } else if (exception instanceof RemotingTimeoutException) { + mqClientException.setResponseCode(ClientErrorCode.ACCESS_BROKER_TIMEOUT); + } else if (exception instanceof MQClientException) { + mqClientException.setResponseCode(ClientErrorCode.BROKER_NOT_EXIST_EXCEPTION); + } + + throw mqClientException; + } + + validateNameServerSetting(); + + throw new MQClientException("No route info of this topic: " + msg.getTopic() + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), + null).setResponseCode(ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION); + } + + private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { + TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); + if (null == topicPublishInfo || !topicPublishInfo.ok()) { + this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + } + + if (topicPublishInfo.isHaveTopicRouterInfo() || topicPublishInfo.ok()) { + return topicPublishInfo; + } else { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + return topicPublishInfo; + } + } + + private SendResult sendKernelImpl(final Message msg, + final MessageQueue mq, + final CommunicationMode communicationMode, + final SendCallback sendCallback, + final TopicPublishInfo topicPublishInfo, + final long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + tryToFindTopicPublishInfo(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + SendMessageContext context = null; + if (brokerAddr != null) { + brokerAddr = MixAll.brokerVIPChannel(this.defaultMQProducer.isSendMessageWithVIPChannel(), brokerAddr); + + byte[] prevBody = msg.getBody(); + try { + //for MessageBatch,ID has been set in the generating process + if (!(msg instanceof MessageBatch)) { + MessageClientIDSetter.setUniqID(msg); + } + + boolean topicWithNamespace = false; + if (null != this.mQClientFactory.getClientConfig().getNamespace()) { + msg.setInstanceId(this.mQClientFactory.getClientConfig().getNamespace()); + topicWithNamespace = true; + } + + int sysFlag = 0; + boolean msgBodyCompressed = false; + if (this.tryToCompressMessage(msg)) { + sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + msgBodyCompressed = true; + } + + final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + } + + if (hasCheckForbiddenHook()) { + CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); + checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); + checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); + checkForbiddenContext.setCommunicationMode(communicationMode); + checkForbiddenContext.setBrokerAddr(brokerAddr); + checkForbiddenContext.setMessage(msg); + checkForbiddenContext.setMq(mq); + checkForbiddenContext.setUnitMode(this.isUnitMode()); + this.executeCheckForbiddenHook(checkForbiddenContext); + } + + if (this.hasSendMessageHook()) { + context = new SendMessageContext(); + context.setProducer(this); + context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + context.setCommunicationMode(communicationMode); + context.setBornHost(this.defaultMQProducer.getClientIP()); + context.setBrokerAddr(brokerAddr); + context.setMessage(msg); + context.setMq(mq); + context.setNamespace(this.defaultMQProducer.getNamespace()); + String isTrans = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (isTrans != null && isTrans.equals("true")) { + context.setMsgType(MessageType.Trans_Msg_Half); + } + + if (msg.getProperty("__STARTDELIVERTIME") != null || msg.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null) { + context.setMsgType(MessageType.Delay_Msg); + } + this.executeSendMessageHookBefore(context); + } + + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); + requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setSysFlag(sysFlag); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + requestHeader.setReconsumeTimes(0); + requestHeader.setUnitMode(this.isUnitMode()); + requestHeader.setBatch(msg instanceof MessageBatch); + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); + if (reconsumeTimes != null) { + requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); + } + + String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(msg); + if (maxReconsumeTimes != null) { + requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); + } + } + + SendResult sendResult = null; + switch (communicationMode) { + case ASYNC: + Message tmpMessage = msg; + boolean messageCloned = false; + if (msgBodyCompressed) { + //If msg body was compressed, msgbody should be reset using prevBody. + //Clone new message using commpressed message body and recover origin massage. + //Fix bug:https://github.com/apache/rocketmq-externals/issues/66 + tmpMessage = MessageAccessor.cloneMessage(msg); + messageCloned = true; + msg.setBody(prevBody); + } + + if (topicWithNamespace) { + if (!messageCloned) { + tmpMessage = MessageAccessor.cloneMessage(msg); + messageCloned = true; + } + msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); + } + + long costTimeAsync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeAsync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( + brokerAddr, + mq.getBrokerName(), + tmpMessage, + requestHeader, + timeout - costTimeAsync, + communicationMode, + sendCallback, + topicPublishInfo, + this.mQClientFactory, + this.defaultMQProducer.getRetryTimesWhenSendAsyncFailed(), + context, + this); + break; + case ONEWAY: + case SYNC: + long costTimeSync = System.currentTimeMillis() - beginStartTime; + if (timeout < costTimeSync) { + throw new RemotingTooMuchRequestException("sendKernelImpl call timeout"); + } + sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage( + brokerAddr, + mq.getBrokerName(), + msg, + requestHeader, + timeout - costTimeSync, + communicationMode, + context, + this); + break; + default: + assert false; + break; + } + + if (this.hasSendMessageHook()) { + context.setSendResult(sendResult); + this.executeSendMessageHookAfter(context); + } + + return sendResult; + } catch (RemotingException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (MQBrokerException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } catch (InterruptedException e) { + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } finally { + msg.setBody(prevBody); + msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultMQProducer.getNamespace())); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + private boolean tryToCompressMessage(final Message msg) { + if (msg instanceof MessageBatch) { + //batch dose not support compressing right now + return false; + } + byte[] body = msg.getBody(); + if (body != null) { + if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { + try { + byte[] data = UtilAll.compress(body, zipCompressLevel); + if (data != null) { + msg.setBody(data); + return true; + } + } catch (IOException e) { + log.error("tryToCompressMessage exception", e); + log.warn(msg.toString()); + } + } + } + + return false; + } + + public boolean hasCheckForbiddenHook() { + return !checkForbiddenHookList.isEmpty(); + } + + public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { + if (hasCheckForbiddenHook()) { + for (CheckForbiddenHook hook : checkForbiddenHookList) { + hook.checkForbidden(context); + } + } + } + + public boolean hasSendMessageHook() { + return !this.sendMessageHookList.isEmpty(); + } + + public void executeSendMessageHookBefore(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageBefore(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookBefore", e); + } + } + } + } + + public void executeSendMessageHookAfter(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageAfter(context); + } catch (Throwable e) { + log.warn("failed to executeSendMessageHookAfter", e); + } + } + } + } + + /** + * DEFAULT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * KERNEL SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueue mq) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, mq, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueue mq, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("call timeout"); + } + + return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null, null, timeout); + } + + /** + * KERNEL ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, mq, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param mq + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + try { + makeSureStateOK(); + Validators.checkMessage(msg, defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } catch (Exception e) { + sendCallback.onException(e); + } + + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("executor rejected ", e); + } + + } + + /** + * KERNEL ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, + MessageQueue mq) throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + /** + * SELECT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, selector, arg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null, timeout); + } + + private SendResult sendSelectImpl( + Message msg, + MessageQueueSelector selector, + Object arg, + final CommunicationMode communicationMode, + final SendCallback sendCallback, final long timeout + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + long beginStartTime = System.currentTimeMillis(); + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + try { + List messageQueueList = + mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); + Message userMessage = MessageAccessor.cloneMessage(msg); + String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); + userMessage.setTopic(userTopic); + + mq = mQClientFactory.getClientConfig().queueWithNamespace(selector.select(messageQueueList, userMessage, arg)); + } catch (Throwable e) { + throw new MQClientException("select message queue throwed exception.", e); + } + + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout < costTime) { + throw new RemotingTooMuchRequestException("sendSelectImpl call timeout"); + } + if (mq != null) { + return this.sendKernelImpl(msg, mq, communicationMode, sendCallback, null, timeout - costTime); + } else { + throw new MQClientException("select message queue return null.", null); + } + } + + validateNameServerSetting(); + throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); + } + + /** + * SELECT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + send(msg, selector, arg, sendCallback, this.defaultMQProducer.getSendMsgTimeout()); + } + + /** + * It will be removed at 4.4.0 cause for exception handling and the wrong Semantics of timeout. A new one will be + * provided in next version + * + * @param msg + * @param selector + * @param arg + * @param sendCallback + * @param timeout the sendCallback will be invoked at most time + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + @Deprecated + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback, final long timeout) + throws MQClientException, RemotingException, InterruptedException { + final long beginStartTime = System.currentTimeMillis(); + ExecutorService executor = this.getAsyncSenderExecutor(); + try { + executor.submit(new Runnable() { + @Override + public void run() { + long costTime = System.currentTimeMillis() - beginStartTime; + if (timeout > costTime) { + try { + try { + sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, + timeout - costTime); + } catch (MQBrokerException e) { + throw new MQClientException("unknownn exception", e); + } + } catch (Exception e) { + sendCallback.onException(e); + } + } else { + sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); + } + } + + }); + } catch (RejectedExecutionException e) { + throw new MQClientException("exector rejected ", e); + } + } + + /** + * SELECT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null, this.defaultMQProducer.getSendMsgTimeout()); + } catch (MQBrokerException e) { + throw new MQClientException("unknown exception", e); + } + } + + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter localTransactionExecuter, final Object arg) + throws MQClientException { + TransactionListener transactionListener = getCheckListener(); + if (null == localTransactionExecuter && null == transactionListener) { + throw new MQClientException("tranExecutor is null", null); + } + + // ignore DelayTimeLevel parameter + if (msg.getDelayTimeLevel() != 0) { + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + } + + Validators.checkMessage(msg, this.defaultMQProducer); + + SendResult sendResult = null; + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, this.defaultMQProducer.getProducerGroup()); + try { + sendResult = this.send(msg); + } catch (Exception e) { + throw new MQClientException("send message Exception", e); + } + + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable localException = null; + switch (sendResult.getSendStatus()) { + case SEND_OK: { + try { + if (sendResult.getTransactionId() != null) { + msg.putUserProperty("__transactionId__", sendResult.getTransactionId()); + } + String transactionId = msg.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (null != transactionId && !"".equals(transactionId)) { + msg.setTransactionId(transactionId); + } + if (null != localTransactionExecuter) { + localTransactionState = localTransactionExecuter.executeLocalTransactionBranch(msg, arg); + } else if (transactionListener != null) { + log.debug("Used new transaction API"); + localTransactionState = transactionListener.executeLocalTransaction(msg, arg); + } + if (null == localTransactionState) { + localTransactionState = LocalTransactionState.UNKNOW; + } + + if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { + log.info("executeLocalTransactionBranch return {}", localTransactionState); + log.info(msg.toString()); + } + } catch (Throwable e) { + log.info("executeLocalTransactionBranch exception", e); + log.info(msg.toString()); + localException = e; + } + } + break; + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; + break; + default: + break; + } + + try { + this.endTransaction(sendResult, localTransactionState, localException); + } catch (Exception e) { + log.warn("local transaction execute " + localTransactionState + ", but end broker transaction failed", e); + } + + TransactionSendResult transactionSendResult = new TransactionSendResult(); + transactionSendResult.setSendStatus(sendResult.getSendStatus()); + transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); + transactionSendResult.setMsgId(sendResult.getMsgId()); + transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); + transactionSendResult.setTransactionId(sendResult.getTransactionId()); + transactionSendResult.setLocalTransactionState(localTransactionState); + return transactionSendResult; + } + + /** + * DEFAULT SYNC ------------------------------------------------------- + */ + public SendResult send( + Message msg) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return send(msg, this.defaultMQProducer.getSendMsgTimeout()); + } + + public void endTransaction( + final SendResult sendResult, + final LocalTransactionState localTransactionState, + final Throwable localException) throws RemotingException, MQBrokerException, InterruptedException, UnknownHostException { + final MessageId id; + if (sendResult.getOffsetMsgId() != null) { + id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); + } else { + id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); + } + String transactionId = sendResult.getTransactionId(); + final String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(sendResult.getMessageQueue().getBrokerName()); + EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); + requestHeader.setTransactionId(transactionId); + requestHeader.setCommitLogOffset(id.getOffset()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_COMMIT_TYPE); + break; + case ROLLBACK_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE); + break; + case UNKNOW: + requestHeader.setCommitOrRollback(MessageSysFlag.TRANSACTION_NOT_TYPE); + break; + default: + break; + } + + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); + requestHeader.setMsgId(sendResult.getMsgId()); + String remark = localException != null ? ("executeLocalTransactionBranch exception: " + localException.toString()) : null; + this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(brokerAddr, requestHeader, remark, + this.defaultMQProducer.getSendMsgTimeout()); + } + + public void setCallbackExecutor(final ExecutorService callbackExecutor) { + this.mQClientFactory.getMQClientAPIImpl().getRemotingClient().setCallbackExecutor(callbackExecutor); + } + + public ExecutorService getAsyncSenderExecutor() { + return null == asyncSenderExecutor ? defaultAsyncSenderExecutor : asyncSenderExecutor; + } + + public void setAsyncSenderExecutor(ExecutorService asyncSenderExecutor) { + this.asyncSenderExecutor = asyncSenderExecutor; + } + + public SendResult send(Message msg, + long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null, timeout); + } + + public ConcurrentMap getTopicPublishInfoTable() { + return topicPublishInfoTable; + } + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + public ServiceState getServiceState() { + return serviceState; + } + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + public long[] getNotAvailableDuration() { + return this.mqFaultStrategy.getNotAvailableDuration(); + } + + public void setNotAvailableDuration(final long[] notAvailableDuration) { + this.mqFaultStrategy.setNotAvailableDuration(notAvailableDuration); + } + + public long[] getLatencyMax() { + return this.mqFaultStrategy.getLatencyMax(); + } + + public void setLatencyMax(final long[] latencyMax) { + this.mqFaultStrategy.setLatencyMax(latencyMax); + } + + public boolean isSendLatencyFaultEnable() { + return this.mqFaultStrategy.isSendLatencyFaultEnable(); + } + + public void setSendLatencyFaultEnable(final boolean sendLatencyFaultEnable) { + this.mqFaultStrategy.setSendLatencyFaultEnable(sendLatencyFaultEnable); + } + + public DefaultMQProducer getDefaultMQProducer() { + return defaultMQProducer; + } } From b3f8008d889a26070c353e589dbace1eb0024e0e Mon Sep 17 00:00:00 2001 From: wqliang Date: Fri, 18 Oct 2019 15:31:08 +0800 Subject: [PATCH 41/73] check transferOK to reduce unnecessary wait --- .../main/java/org/apache/rocketmq/store/ha/HAService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index 2d02d43169..fd24a75239 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -282,12 +282,9 @@ public class HAService { boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); long waitUntillWhen = HAService.this.defaultMessageStore.getSystemClock().now() + HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(); - while (HAService.this.defaultMessageStore.getSystemClock().now() < waitUntillWhen) { + while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntillWhen) { this.notifyTransferObject.waitForRunning(1000); transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); - if (transferOK) { - break; - } } if (!transferOK) { From 93ab99e0596a8d6db279fd18a82e3a432988258f Mon Sep 17 00:00:00 2001 From: wqliang Date: Fri, 18 Oct 2019 17:20:19 +0800 Subject: [PATCH 42/73] correct spelling mistakes --- .../src/main/java/org/apache/rocketmq/store/ha/HAService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java index fd24a75239..3035c575cf 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java @@ -280,9 +280,9 @@ public class HAService { if (!this.requestsRead.isEmpty()) { for (CommitLog.GroupCommitRequest req : this.requestsRead) { boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); - long waitUntillWhen = HAService.this.defaultMessageStore.getSystemClock().now() + long waitUntilWhen = HAService.this.defaultMessageStore.getSystemClock().now() + HAService.this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout(); - while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntillWhen) { + while (!transferOK && HAService.this.defaultMessageStore.getSystemClock().now() < waitUntilWhen) { this.notifyTransferObject.waitForRunning(1000); transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); } From fdbd3cb85108e3f63f9b5737983866d4425f2fbe Mon Sep 17 00:00:00 2001 From: huangli Date: Sat, 12 Oct 2019 11:20:48 +0800 Subject: [PATCH 43/73] Optimise transaction check performance: fix problem that processed opOffset not add to doneOpOffset --- .../transaction/queue/TransactionalMessageServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index e1549b1517..09c9981efe 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -159,7 +159,8 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ } if (removeMap.containsKey(i)) { log.info("Half offset {} has been committed/rolled back", i); - removeMap.remove(i); + Long removedOpOffset = removeMap.remove(i); + doneOpOffset.add(removedOpOffset); } else { GetResult getResult = getHalfMsg(messageQueue, i); MessageExt msgExt = getResult.getMsg(); From bafd621868737d63a917c4b9ea4f6e12cd134d0f Mon Sep 17 00:00:00 2001 From: huangli Date: Tue, 15 Oct 2019 09:25:10 +0800 Subject: [PATCH 44/73] change log level, optimise performance of endTransaction and txCheckTask --- .../rocketmq/broker/processor/EndTransactionProcessor.java | 2 +- .../transaction/queue/TransactionalMessageServiceImpl.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java index c9e85ed1a5..1d5943d431 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/EndTransactionProcessor.java @@ -55,7 +55,7 @@ public class EndTransactionProcessor implements NettyRequestProcessor { final RemotingCommand response = RemotingCommand.createResponseCommand(null); final EndTransactionRequestHeader requestHeader = (EndTransactionRequestHeader)request.decodeCommandCustomHeader(EndTransactionRequestHeader.class); - LOGGER.info("Transaction request:{}", requestHeader); + LOGGER.debug("Transaction request:{}", requestHeader); if (BrokerRole.SLAVE == brokerController.getMessageStoreConfig().getBrokerRole()) { response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); LOGGER.warn("Message store is slave mode, so end transaction is forbidden. "); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java index 09c9981efe..71b575eec0 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java @@ -224,7 +224,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ listener.resolveHalfMsg(msgExt); } else { pullResult = fillOpRemoveMap(removeMap, opQueue, pullResult.getNextBeginOffset(), halfOffset, doneOpOffset); - log.info("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, + log.debug("The miss offset:{} in messageQueue:{} need to get more opMsg, result is:{}", i, messageQueue, pullResult); continue; } @@ -293,7 +293,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ } for (MessageExt opMessageExt : opMsg) { Long queueOffset = getLong(new String(opMessageExt.getBody(), TransactionalMessageUtil.charset)); - log.info("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), + log.debug("Topic: {} tags: {}, OpOffset: {}, HalfOffset: {}", opMessageExt.getTopic(), opMessageExt.getTags(), opMessageExt.getQueueOffset(), queueOffset); if (TransactionalMessageUtil.REMOVETAG.equals(opMessageExt.getTags())) { if (queueOffset < miniOffset) { @@ -461,7 +461,7 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ @Override public boolean deletePrepareMessage(MessageExt msgExt) { if (this.transactionalMessageBridge.putOpMessage(msgExt, TransactionalMessageUtil.REMOVETAG)) { - log.info("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); + log.debug("Transaction op message write successfully. messageId={}, queueId={} msgExt:{}", msgExt.getMsgId(), msgExt.getQueueId(), msgExt); return true; } else { log.error("Transaction op message write failed. messageId is {}, queueId is {}", msgExt.getMsgId(), msgExt.getQueueId()); From 6fb66c1f4c8fea6daed15011923fd55665338a31 Mon Sep 17 00:00:00 2001 From: huangli Date: Sat, 19 Oct 2019 23:16:10 +0800 Subject: [PATCH 45/73] Fix problem when queue of transaction check ExecutorService is full. --- .../transaction/AbstractTransactionalMessageCheckListener.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java index 62507cdfdb..35d811207b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java @@ -30,6 +30,7 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; import java.util.concurrent.TimeUnit; public abstract class AbstractTransactionalMessageCheckListener { @@ -48,7 +49,7 @@ public abstract class AbstractTransactionalMessageCheckListener { thread.setName("Transaction-msg-check-thread"); return thread; } - }); + }, new CallerRunsPolicy()); public AbstractTransactionalMessageCheckListener() { } From 01827c8dd3a4e375cdea4e802f93c802ec43ea83 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Mon, 21 Oct 2019 11:37:30 +0800 Subject: [PATCH 46/73] remove unused code and fix typo --- .../broker/processor/ReplyMessageProcessorTest.java | 3 +-- .../rocketmq/client/impl/ClientRemotingProcessor.java | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java index 94fd8f560d..85c775040f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessorTest.java @@ -71,8 +71,7 @@ public class ReplyMessageProcessorTest { private MessageStore messageStore; @Mock private Channel channel; - @Mock - private TransactionalMessageService transactionMsgService; + private String topic = "FooBar"; private String group = "FooBarGroup"; private ClientChannelInfo clientInfo; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index 5cac5e832a..c57e49bb61 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -83,7 +83,7 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { return this.consumeMessageDirectly(ctx, request); case RequestCode.PUSH_REPLY_MESSAGE_TO_CLIENT: - return this.receiveReplyMssage(ctx, request); + return this.receiveReplyMessage(ctx, request); default: break; } @@ -222,7 +222,7 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { return response; } - private RemotingCommand receiveReplyMssage(ChannelHandlerContext ctx, + private RemotingCommand receiveReplyMessage(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); @@ -293,7 +293,9 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { requestResponseFuture.putResponseMessage(replyMsg); } } else { - log.warn(String.format("receive reply message, but not matched any request, CorrelationId: %s", correlationId)); + String bornHost = replyMsg.getBornHostString(); + log.warn(String.format("receive reply message, but not matched any request, CorrelationId: %s , reply from host: %s", + correlationId, bornHost)); } } } From 49f0a0ec066705d479ec2859cdf4870b0c689fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=92=E6=9C=A8=E6=81=AD?= Date: Tue, 22 Oct 2019 14:37:02 +0800 Subject: [PATCH 47/73] Update RocketMQ_Example.md --- docs/cn/RocketMQ_Example.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/cn/RocketMQ_Example.md b/docs/cn/RocketMQ_Example.md index d298db8420..09acb0de97 100644 --- a/docs/cn/RocketMQ_Example.md +++ b/docs/cn/RocketMQ_Example.md @@ -66,7 +66,11 @@ public class AsyncProducer { // 启动Producer实例 producer.start(); producer.setRetryTimesWhenSendAsyncFailed(0); - for (int i = 0; i < 100; i++) { + + int messageCount = 100; + // 根据消息数量实例化倒计时计算器 + final CountDownLatch2 countDownLatch = new CountDownLatch2(messageCount); + for (int i = 0; i < messageCount; i++) { final int index = i; // 创建消息，并指定Topic，Tag和消息体 Message msg = new Message("TopicTest", @@ -87,6 +91,8 @@ public class AsyncProducer { } }); } + // 等待5s + countDownLatch.await(5, TimeUnit.SECONDS); // 如果不再发送消息，关闭Producer实例。 producer.shutdown(); } From 520a4c4e5b98d8614e6b975de1c5a2149cfdf28c Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Wed, 23 Oct 2019 20:40:00 +0800 Subject: [PATCH 48/73] modify unit test of producer --- .../producer/DefaultMQProducerTest.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 2e23eaa7f3..35f65e36c2 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -107,16 +107,20 @@ public class DefaultMQProducerTest { nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenReturn(createSendResult(SendStatus.SEND_OK)); - when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), - any(SendCallback.class), nullable(TopicPublishInfo.class), any(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) .thenAnswer(new Answer() { @Override public Object answer(InvocationOnMock invocation) throws Throwable { - Object[] args = invocation.getArguments(); - SendCallback callback = (SendCallback) args[6]; - callback.onSuccess(createSendResult(SendStatus.SEND_OK)); - return new SendResult(); + CommunicationMode mode = invocation.getArgument(5); + switch (mode) { + case SYNC: + return createSendResult(SendStatus.SEND_OK); + case ONEWAY: + case ASYNC: + SendCallback callback = invocation.getArgument(6); + callback.onSuccess(createSendResult(SendStatus.SEND_OK)); + return null; + } + return null; } }); } @@ -244,6 +248,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { + final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(1); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @@ -252,6 +257,7 @@ public class DefaultMQProducerTest { assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); + cc.incrementAndGet(); countDownLatch.countDown(); } @@ -260,6 +266,7 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + assertThat(cc.get()).isEqualTo(1); } @Test From 119feeedd40a109ead1e777d15f75e6ec1bd02b8 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Thu, 24 Oct 2019 10:19:40 +0800 Subject: [PATCH 49/73] use RemotingUtil to parse address --- .../client/impl/ClientRemotingProcessor.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index c57e49bb61..b093b9531e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -51,6 +51,7 @@ import org.apache.rocketmq.common.protocol.header.ResetOffsetRequestHeader; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.common.RemotingUtil; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -236,19 +237,11 @@ public class ClientRemotingProcessor implements NettyRequestProcessor { msg.setStoreTimestamp(requestHeader.getStoreTimestamp()); if (requestHeader.getBornHost() != null) { - String[] bornHostArr = requestHeader.getBornHost().split("/"); - String bornHost/*ip:port*/ = bornHostArr[bornHostArr.length - 1]; - String[] host = bornHost.split(":"); - if (host.length == 2) - msg.setBornHost(new InetSocketAddress(host[0], Integer.parseInt(host[1]))); + msg.setBornHost(RemotingUtil.string2SocketAddress(requestHeader.getBornHost())); } if (requestHeader.getStoreHost() != null) { - String[] storeHostArr = requestHeader.getStoreHost().split("/"); - String storeHost = storeHostArr[storeHostArr.length - 1]; - String[] host = storeHost.split(":"); - if (host.length == 2) - msg.setStoreHost(new InetSocketAddress(host[0], Integer.parseInt(host[1]))); + msg.setStoreHost(RemotingUtil.string2SocketAddress(requestHeader.getStoreHost())); } byte[] body = request.getBody(); From a314e17b751f92b4935fb650963c9ee02ce8aa48 Mon Sep 17 00:00:00 2001 From: qqeasonchen Date: Thu, 24 Oct 2019 10:46:00 +0800 Subject: [PATCH 50/73] remove unused import --- .../org/apache/rocketmq/client/impl/ClientRemotingProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java index b093b9531e..5861bc4baa 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/ClientRemotingProcessor.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.client.impl; import io.netty.channel.ChannelHandlerContext; import java.io.IOException; -import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; From 1fdd43c27151ec7d8d1403777bcb1ffdd6d3b0e4 Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Thu, 24 Oct 2019 11:28:37 +0800 Subject: [PATCH 51/73] modify unit test of producer --- .../rocketmq/client/producer/DefaultMQProducerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 35f65e36c2..cb32bda8ac 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -185,7 +185,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicInteger cc = new AtomicInteger(0); +// final AtomicInteger cc = new AtomicInteger(0); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override @@ -194,7 +194,7 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); - cc.incrementAndGet(); +// cc.incrementAndGet(); } @Override @@ -202,7 +202,7 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(1); +// assertThat(cc.get()).isEqualTo(1); } @Test From 761508a9bd24b1f8d310009023bbfe2f4bdac63a Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Thu, 24 Oct 2019 20:52:34 +0800 Subject: [PATCH 52/73] modify unit test of producer --- .../rocketmq/client/producer/DefaultMQProducerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index cb32bda8ac..35f65e36c2 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -185,7 +185,7 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); -// final AtomicInteger cc = new AtomicInteger(0); + final AtomicInteger cc = new AtomicInteger(0); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override @@ -194,7 +194,7 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); -// cc.incrementAndGet(); + cc.incrementAndGet(); } @Override @@ -202,7 +202,7 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); -// assertThat(cc.get()).isEqualTo(1); + assertThat(cc.get()).isEqualTo(1); } @Test From 71a57f5033de5212e3329eaa20ddf6f092925e05 Mon Sep 17 00:00:00 2001 From: lwl Date: Fri, 25 Oct 2019 07:48:12 +0800 Subject: [PATCH 53/73] modify docs-cn description error --- docs/cn/README | 0 docs/cn/features.md | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/cn/README diff --git a/docs/cn/README b/docs/cn/README new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/cn/features.md b/docs/cn/features.md index 859e0f84a3..e7b63864e9 100644 --- a/docs/cn/features.md +++ b/docs/cn/features.md @@ -16,7 +16,7 @@ RocketMQ的消费者可以根据Tag进行消息过滤，也支持自定义属性过滤。消息过滤目前是在Broker端实现的，优点是减少了对于Consumer无用消息的网络传输，缺点是增加了Broker的负担、而且实现相对复杂。 ## 4 消息可靠性 RocketMQ支持消息的高可靠，影响消息可靠性的几种情况： -1) Broker正常关闭 +1) Broker非正常关闭 2) Broker异常Crash 3) OS Crash 4) 机器掉电，但是能立即恢复供电情况 From 076a96df62f8ef5c8e5211d5d385da48362e3251 Mon Sep 17 00:00:00 2001 From: xiang Date: Mon, 28 Oct 2019 00:22:30 +0800 Subject: [PATCH 54/73] fix cacography --- docs/cn/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cn/README.md b/docs/cn/README.md index c8bdc8dc0d..38554a3652 100644 --- a/docs/cn/README.md +++ b/docs/cn/README.md @@ -1,7 +1,7 @@ Apache RocketMQ开发者指南 -------- -##### 这个开发者指南是帮忙您快速了解,并使用 Apache RocketMQ +##### 这个开发者指南是帮助您快速了解,并使用 Apache RocketMQ ### 1. 概念和特性 From dee6c7ba40ee11d042506398696c9f0197535d62 Mon Sep 17 00:00:00 2001 From: duhenglucky Date: Thu, 31 Oct 2019 11:43:30 +0800 Subject: [PATCH 55/73] fix(UtilAll) exclude local link ipv6 address --- .../org/apache/rocketmq/common/UtilAll.java | 17 ++++++----------- .../org/apache/rocketmq/common/UtilAllTest.java | 14 ++++++++------ .../message/MessageClientIDSetterTest.java | 3 +-- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java index 85a4cde121..222f697eb5 100644 --- a/common/src/main/java/org/apache/rocketmq/common/UtilAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/UtilAll.java @@ -440,13 +440,11 @@ public class UtilAll { return false; } - public static boolean isInternalV6IP(byte[] ip) { - if (ip.length != 16) { - throw new RuntimeException("illegal ipv6 bytes"); - } - - //FEC0:0000:0000:0000:0000:0000:0000:0000/10 - if (ip[0] == (byte) 254 && ip[1] >= (byte) 192) { + public static boolean isInternalV6IP(InetAddress inetAddr) { + if (inetAddr.isAnyLocalAddress() // Wild card ipv6 + || inetAddr.isLinkLocalAddress() // Single broadcast ipv6 address: fe80:xx:xx... + || inetAddr.isLoopbackAddress() //Loopback ipv6 address + || inetAddr.isSiteLocalAddress()) { // Site local ipv6 address: fec0:xx:xx... return true; } return false; @@ -457,9 +455,6 @@ public class UtilAll { throw new RuntimeException("illegal ipv4 bytes"); } -// if (ip[0] == (byte)30 && ip[1] == (byte)10 && ip[2] == (byte)163 && ip[3] == (byte)120) { -// } - if (ip[0] >= (byte) 1 && ip[0] <= (byte) 126) { if (ip[1] == (byte) 1 && ip[2] == (byte) 1 && ip[3] == (byte) 1) { return false; @@ -550,7 +545,7 @@ public class UtilAll { byte[] ipByte = ip.getAddress(); if (ipByte.length == 16) { if (ipV6Check(ipByte)) { - if (!isInternalV6IP(ipByte)) { + if (!isInternalV6IP(ip)) { return ipByte; } else if (internalIP == null) { internalIP = ipByte; diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java index 3991367330..b854099a5b 100644 --- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java @@ -17,6 +17,8 @@ package org.apache.rocketmq.common; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Properties; import org.junit.Test; @@ -99,12 +101,12 @@ public class UtilAllTest { } @Test - public void testIPv6Check() { - byte[] nonInternalIp = UtilAll.string2bytes("24084004018081003FAA1DDE2B3F898A"); - byte[] internalIp = UtilAll.string2bytes("FEC0000000000000000000000000FFFF"); - assertThat(UtilAll.isInternalV6IP(nonInternalIp)).isFalse(); - assertThat(UtilAll.isInternalV6IP(internalIp)).isTrue(); - assertThat(UtilAll.ipToIPv6Str(nonInternalIp).toUpperCase()).isEqualTo("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A"); + public void testIPv6Check() throws UnknownHostException { + InetAddress nonInternal = InetAddress.getByName("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A"); + InetAddress internal = InetAddress.getByName("FE80:0000:0000:0000:0000:0000:0000:FFFF"); + assertThat(UtilAll.isInternalV6IP(nonInternal)).isFalse(); + assertThat(UtilAll.isInternalV6IP(internal)).isTrue(); + assertThat(UtilAll.ipToIPv6Str(nonInternal.getAddress()).toUpperCase()).isEqualTo("2408:4004:0180:8100:3FAA:1DDE:2B3F:898A"); } static class DemoConfig { diff --git a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java index 0df75eb30a..1ec6d93cff 100644 --- a/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java +++ b/common/src/test/java/org/apache/rocketmq/common/message/MessageClientIDSetterTest.java @@ -17,9 +17,8 @@ package org.apache.rocketmq.common.message; -import java.util.Calendar; -import java.util.Date; import org.junit.Test; + import static org.assertj.core.api.Assertions.assertThat; public class MessageClientIDSetterTest { From 37cbf9f80cfa60491ab6c5ecdb8099d008abcfb7 Mon Sep 17 00:00:00 2001 From: zhenghu Date: Mon, 4 Nov 2019 16:57:26 +0800 Subject: [PATCH 56/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/transaction/queue/TransactionalMessageBridge.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 84a62761bd..ef92fbcd00 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -176,7 +176,9 @@ public class TransactionalMessageBridge { List messageBufferList = getMessageResult.getMessageBufferList(); for (ByteBuffer bb : messageBufferList) { MessageExt msgExt = MessageDecoder.decode(bb); - foundList.add(msgExt); + if(msgExt != null){ + foundList.add(msgExt); + } } } finally { From e8a0fea48fbf79f66ed1cb5a44419931badb93db Mon Sep 17 00:00:00 2001 From: zhenghu Date: Mon, 4 Nov 2019 17:18:17 +0800 Subject: [PATCH 57/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/transaction/queue/TransactionalMessageBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index ef92fbcd00..b1a88b5f7f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -176,7 +176,7 @@ public class TransactionalMessageBridge { List messageBufferList = getMessageResult.getMessageBufferList(); for (ByteBuffer bb : messageBufferList) { MessageExt msgExt = MessageDecoder.decode(bb); - if(msgExt != null){ + if (msgExt != null) { foundList.add(msgExt); } } From d2345731b9998603a15f1fdc2e8ee96d1e9d439d Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 00:53:34 +0800 Subject: [PATCH 58/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0test=E8=A6=86=E7=9B=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queue/TransactionalMessageBridge.java | 3 +++ .../queue/TransactionalMessageBridgeTest.java | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index b1a88b5f7f..8e283758d4 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -131,6 +131,9 @@ public class TransactionalMessageBridge { this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize()); this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); + if ( foundList == null || foundList.size() == 0 ) { + break; + } this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, this.brokerController.getMessageStore().now() - foundList.get(foundList.size() - 1) .getStoreTimestamp()); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index b1c669c993..f2636b7b14 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -167,6 +167,24 @@ public class TransactionalMessageBridgeTest { assertThat(messageExt).isNotNull(); } + @Test + public void testGetHalfMessageStatusFound() { + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(createGetMessageResult(GetMessageStatus.FOUND)); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result.getPullStatus()).isEqualTo(PullStatus.FOUND); + } + + @Test + public void testGetHalfMessageNull() { + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(null); + PullResult result = transactionBridge.getHalfMessage(0, 0, 1); + assertThat(result.getPullStatus()).isNull(); + } + private GetMessageResult createGetMessageResult(GetMessageStatus status) { GetMessageResult getMessageResult = new GetMessageResult(); getMessageResult.setStatus(status); From 159d28d5f8b0d917934a56cda2cb72dbd3cdcc9b Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 01:22:37 +0800 Subject: [PATCH 59/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0test=E8=A6=86=E7=9B=96=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/transaction/queue/TransactionalMessageBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index 8e283758d4..e14ed05a65 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -131,7 +131,7 @@ public class TransactionalMessageBridge { this.brokerController.getBrokerStatsManager().incGroupGetSize(group, topic, getMessageResult.getBufferTotalSize()); this.brokerController.getBrokerStatsManager().incBrokerGetNums(getMessageResult.getMessageCount()); - if ( foundList == null || foundList.size() == 0 ) { + if (foundList == null || foundList.size() == 0) { break; } this.brokerController.getBrokerStatsManager().recordDiskFallBehindTime(group, topic, queueId, From a4f62104c8df52523dd760ccb3560fad3fc8d0b2 Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 01:45:24 +0800 Subject: [PATCH 60/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0test=E8=A6=86=E7=9B=96=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../queue/TransactionalMessageBridgeTest.java | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index f2636b7b14..ee27204fda 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -62,7 +62,7 @@ public class TransactionalMessageBridgeTest { @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), - new NettyClientConfig(), new MessageStoreConfig()); + new NettyClientConfig(), new MessageStoreConfig()); @Mock private MessageStore messageStore; @@ -82,7 +82,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutHalfMessage() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner()); assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); } @@ -96,7 +96,7 @@ public class TransactionalMessageBridgeTest { @Test public void testFetchConsumeOffset() { MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), - 0); + 0); long offset = transactionBridge.fetchConsumeOffset(mq); assertThat(offset).isGreaterThan(-1); } @@ -104,20 +104,24 @@ public class TransactionalMessageBridgeTest { @Test public void updateConsumeOffset() { MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), - 0); + 0); transactionBridge.updateConsumeOffset(mq, 0); } @Test public void testGetHalfMessage() { - when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); PullResult result = transactionBridge.getHalfMessage(0, 0, 1); assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); } @Test public void testGetOpMessage() { - when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + when(messageStore + .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) + .thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); PullResult result = transactionBridge.getOpMessage(0, 0, 1); assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); } @@ -125,7 +129,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutMessageReturnResult() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner()); assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); } @@ -133,7 +137,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutMessage() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); Boolean success = transactionBridge.putMessage(createMessageBrokerInner()); assertThat(success).isEqualTo(true); } @@ -143,9 +147,9 @@ public class TransactionalMessageBridgeTest { MessageExt messageExt = createMessageBrokerInner(); final String offset = "123456789"; MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt); - MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, offset); assertThat(msgInner).isNotNull(); - Map properties = msgInner.getProperties(); + Map properties = msgInner.getProperties(); assertThat(properties).isNotNull(); String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); assertThat(resOffset).isEqualTo(offset); @@ -157,11 +161,11 @@ public class TransactionalMessageBridgeTest { MessageExt messageExt = new MessageExt(); long bornTimeStamp = messageExt.getBornTimestamp(); MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt); - assertThat( messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); + assertThat(messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); } @Test - public void testLookMessageByOffset(){ + public void testLookMessageByOffset() { when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); MessageExt messageExt = transactionBridge.lookMessageByOffset(123); assertThat(messageExt).isNotNull(); @@ -182,7 +186,7 @@ public class TransactionalMessageBridgeTest { .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) .thenReturn(null); PullResult result = transactionBridge.getHalfMessage(0, 0, 1); - assertThat(result.getPullStatus()).isNull(); + assertThat(result).isNull(); } private GetMessageResult createGetMessageResult(GetMessageStatus status) { From 9b9d2bd74caf13dd86abc1e9b03908ffbf947026 Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 01:48:00 +0800 Subject: [PATCH 61/73] =?UTF-8?q?Revert=20"[#1568]=20rocketmq=20need=20enh?= =?UTF-8?q?ance=20stability=20when=20commitlog=20broken=20=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0test=E8=A6=86=E7=9B=96=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E7=BC=96=E7=A0=81=E6=A0=BC=E5=BC=8F"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit a4f62104 --- .../queue/TransactionalMessageBridgeTest.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index ee27204fda..f2636b7b14 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -62,7 +62,7 @@ public class TransactionalMessageBridgeTest { @Spy private BrokerController brokerController = new BrokerController(new BrokerConfig(), new NettyServerConfig(), - new NettyClientConfig(), new MessageStoreConfig()); + new NettyClientConfig(), new MessageStoreConfig()); @Mock private MessageStore messageStore; @@ -82,7 +82,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutHalfMessage() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); PutMessageResult result = transactionBridge.putHalfMessage(createMessageBrokerInner()); assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); } @@ -96,7 +96,7 @@ public class TransactionalMessageBridgeTest { @Test public void testFetchConsumeOffset() { MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), - 0); + 0); long offset = transactionBridge.fetchConsumeOffset(mq); assertThat(offset).isGreaterThan(-1); } @@ -104,24 +104,20 @@ public class TransactionalMessageBridgeTest { @Test public void updateConsumeOffset() { MessageQueue mq = new MessageQueue(TransactionalMessageUtil.buildOpTopic(), this.brokerController.getBrokerConfig().getBrokerName(), - 0); + 0); transactionBridge.updateConsumeOffset(mq, 0); } @Test public void testGetHalfMessage() { - when(messageStore - .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) - .thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); PullResult result = transactionBridge.getHalfMessage(0, 0, 1); assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); } @Test public void testGetOpMessage() { - when(messageStore - .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) - .thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); + when(messageStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))).thenReturn(createGetMessageResult(GetMessageStatus.NO_MESSAGE_IN_QUEUE)); PullResult result = transactionBridge.getOpMessage(0, 0, 1); assertThat(result.getPullStatus()).isEqualTo(PullStatus.NO_NEW_MSG); } @@ -129,7 +125,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutMessageReturnResult() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); PutMessageResult result = transactionBridge.putMessageReturnResult(createMessageBrokerInner()); assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); } @@ -137,7 +133,7 @@ public class TransactionalMessageBridgeTest { @Test public void testPutMessage() { when(messageStore.putMessage(any(MessageExtBrokerInner.class))).thenReturn(new PutMessageResult - (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); + (PutMessageStatus.PUT_OK, new AppendMessageResult(AppendMessageStatus.PUT_OK))); Boolean success = transactionBridge.putMessage(createMessageBrokerInner()); assertThat(success).isEqualTo(true); } @@ -147,9 +143,9 @@ public class TransactionalMessageBridgeTest { MessageExt messageExt = createMessageBrokerInner(); final String offset = "123456789"; MessageExtBrokerInner msgInner = transactionBridge.renewImmunityHalfMessageInner(messageExt); - MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET, offset); + MessageAccessor.putProperty(msgInner, MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET,offset); assertThat(msgInner).isNotNull(); - Map properties = msgInner.getProperties(); + Map properties = msgInner.getProperties(); assertThat(properties).isNotNull(); String resOffset = properties.get(MessageConst.PROPERTY_TRANSACTION_PREPARED_QUEUE_OFFSET); assertThat(resOffset).isEqualTo(offset); @@ -161,11 +157,11 @@ public class TransactionalMessageBridgeTest { MessageExt messageExt = new MessageExt(); long bornTimeStamp = messageExt.getBornTimestamp(); MessageExt messageExtRes = transactionBridge.renewHalfMessageInner(messageExt); - assertThat(messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); + assertThat( messageExtRes.getBornTimestamp()).isEqualTo(bornTimeStamp); } @Test - public void testLookMessageByOffset() { + public void testLookMessageByOffset(){ when(messageStore.lookMessageByOffset(anyLong())).thenReturn(new MessageExt()); MessageExt messageExt = transactionBridge.lookMessageByOffset(123); assertThat(messageExt).isNotNull(); @@ -186,7 +182,7 @@ public class TransactionalMessageBridgeTest { .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) .thenReturn(null); PullResult result = transactionBridge.getHalfMessage(0, 0, 1); - assertThat(result).isNull(); + assertThat(result.getPullStatus()).isNull(); } private GetMessageResult createGetMessageResult(GetMessageStatus status) { From 903783f9b99374414c036061e762f1db0ed161ee Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 01:51:09 +0800 Subject: [PATCH 62/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0test=E8=A6=86=E7=9B=96=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../transaction/queue/TransactionalMessageBridgeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java index f2636b7b14..ebe887285f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridgeTest.java @@ -182,7 +182,7 @@ public class TransactionalMessageBridgeTest { .getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), ArgumentMatchers.nullable(MessageFilter.class))) .thenReturn(null); PullResult result = transactionBridge.getHalfMessage(0, 0, 1); - assertThat(result.getPullStatus()).isNull(); + assertThat(result).isNull(); } private GetMessageResult createGetMessageResult(GetMessageStatus status) { From f8eb4d80a2d56d6147add795963f4931fa0fa424 Mon Sep 17 00:00:00 2001 From: zhenghu Date: Tue, 5 Nov 2019 14:08:47 +0800 Subject: [PATCH 63/73] =?UTF-8?q?[#1568]=20rocketmq=20need=20enhance=20sta?= =?UTF-8?q?bility=20when=20commitlog=20broken=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9D=9E=E7=A9=BA=E5=88=A4=E6=96=AD=EF=BC=8C=E9=98=B2=E6=AD=A2?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E5=BC=82=E5=B8=B8,=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=BE=AA=E7=8E=AF=E4=BA=8B=E5=8A=A1=20check,=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0test=E8=A6=86=E7=9B=96=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E7=BC=96=E7=A0=81=E6=A0=BC=E5=BC=8F=20=E5=8E=9F=E5=9B=A0?= =?UTF-8?q?=EF=BC=8C=E6=B6=88=E6=81=AF=E5=A4=AA=E9=95=BF=E5=8E=8B=E7=BC=A9?= =?UTF-8?q?=E5=90=8E=E9=87=8D=E5=A4=8D=E8=A7=A3=E5=8E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../broker/transaction/queue/TransactionalMessageBridge.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java index e14ed05a65..45699032f9 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageBridge.java @@ -178,7 +178,7 @@ public class TransactionalMessageBridge { try { List messageBufferList = getMessageResult.getMessageBufferList(); for (ByteBuffer bb : messageBufferList) { - MessageExt msgExt = MessageDecoder.decode(bb); + MessageExt msgExt = MessageDecoder.decode(bb, true, false); if (msgExt != null) { foundList.add(msgExt); } From 34155096bb7baaaed24941c5ca1dcae8397afbb2 Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Tue, 12 Nov 2019 10:15:00 +0800 Subject: [PATCH 64/73] modify unit test of producer --- .../apache/rocketmq/client/producer/DefaultMQProducerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 35f65e36c2..7fb2621c4a 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -201,7 +201,7 @@ public class DefaultMQProducerTest { public void onException(Throwable e) { } }); - countDownLatch.await(3000L, TimeUnit.MILLISECONDS); + countDownLatch.await(20000L, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(1); } From b8e1be8a7749bc2546b678b7d66b7ddd78ef320b Mon Sep 17 00:00:00 2001 From: keranbingaa Date: Tue, 12 Nov 2019 11:12:51 +0800 Subject: [PATCH 65/73] modify unit test of producer --- .../apache/rocketmq/client/producer/DefaultMQProducerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 7fb2621c4a..8e73ba5eae 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -201,7 +201,7 @@ public class DefaultMQProducerTest { public void onException(Throwable e) { } }); - countDownLatch.await(20000L, TimeUnit.MILLISECONDS); + countDownLatch.await(8000L, TimeUnit.MILLISECONDS); assertThat(cc.get()).isEqualTo(1); } From ea09ce093f4aea64ac476ca55e303990a6d43009 Mon Sep 17 00:00:00 2001 From: Eason Chen Date: Wed, 13 Nov 2019 16:07:11 +0800 Subject: [PATCH 66/73] [issue #670]change the way to create newGroupChannelTable (#1412) * change the way to create newGroupChannelTable --- .../apache/rocketmq/broker/client/ProducerManager.java | 10 +++++++++- .../rocketmq/broker/client/ProducerManagerTest.java | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index 61ceae5386..f3eed652ce 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -53,7 +53,15 @@ public class ProducerManager { try { if (this.groupChannelLock.tryLock(LOCK_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS)) { try { - newGroupChannelTable.putAll(groupChannelTable); + Iterator>> iter = groupChannelTable.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry> entry = iter.next(); + String key = entry.getKey(); + HashMap val = entry.getValue(); + HashMap tmp = new HashMap(); + tmp.putAll(val); + newGroupChannelTable.put(key, tmp); + } } finally { groupChannelLock.unlock(); } diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index 08dbb9c754..33995ff2fe 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -87,4 +87,12 @@ public class ProducerManagerTest { assertThat(channelMap).isNull(); } + @Test + public void testGetGroupChannelTable() throws Exception { + producerManager.registerProducer(group, clientInfo); + HashMap oldMap = producerManager.getGroupChannelTable().get(group); + + producerManager.unregisterProducer(group, clientInfo); + assertThat(oldMap.size()).isNotEqualTo(0); + } } \ No newline at end of file From e89989699cc2865229375e7723feef88590d3832 Mon Sep 17 00:00:00 2001 From: wqliang Date: Mon, 18 Nov 2019 13:47:23 +0800 Subject: [PATCH 67/73] [ISSUE #1528]modify testSendMessageAsync_Success (#1599) * modify testSendMessageAsync_Success * revert testSendMessageAsync_Success --- .../producer/DefaultMQProducerTest.java | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java index 818c94a509..d572a2320e 100644 --- a/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/producer/DefaultMQProducerTest.java @@ -111,22 +111,7 @@ public class DefaultMQProducerTest { nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))).thenCallRealMethod(); when(mQClientAPIImpl.sendMessage(anyString(), anyString(), any(Message.class), any(SendMessageRequestHeader.class), anyLong(), any(CommunicationMode.class), nullable(SendCallback.class), nullable(TopicPublishInfo.class), nullable(MQClientInstance.class), anyInt(), nullable(SendMessageContext.class), any(DefaultMQProducerImpl.class))) - .thenAnswer(new Answer() { - @Override - public Object answer(InvocationOnMock invocation) throws Throwable { - CommunicationMode mode = invocation.getArgument(5); - switch (mode) { - case SYNC: - return createSendResult(SendStatus.SEND_OK); - case ONEWAY: - case ASYNC: - SendCallback callback = invocation.getArgument(6); - callback.onSuccess(createSendResult(SendStatus.SEND_OK)); - return null; - } - return null; - } - }); + .thenReturn(createSendResult(SendStatus.SEND_OK)); } @After @@ -189,7 +174,6 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_Success() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { final CountDownLatch countDownLatch = new CountDownLatch(1); - final AtomicInteger cc = new AtomicInteger(0); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(message, new SendCallback() { @Override @@ -198,15 +182,14 @@ public class DefaultMQProducerTest { assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); countDownLatch.countDown(); - cc.incrementAndGet(); } @Override public void onException(Throwable e) { + countDownLatch.countDown(); } }); - countDownLatch.await(8000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(1); + countDownLatch.await(3000L, TimeUnit.MILLISECONDS); } @Test @@ -252,7 +235,6 @@ public class DefaultMQProducerTest { @Test public void testSendMessageAsync_BodyCompressed() throws RemotingException, InterruptedException, MQBrokerException, MQClientException { - final AtomicInteger cc = new AtomicInteger(0); final CountDownLatch countDownLatch = new CountDownLatch(1); when(mQClientAPIImpl.getTopicRouteInfoFromNameServer(anyString(), anyLong())).thenReturn(createTopicRoute()); producer.send(bigMessage, new SendCallback() { @@ -261,7 +243,6 @@ public class DefaultMQProducerTest { assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); assertThat(sendResult.getOffsetMsgId()).isEqualTo("123"); assertThat(sendResult.getQueueOffset()).isEqualTo(456L); - cc.incrementAndGet(); countDownLatch.countDown(); } @@ -270,7 +251,6 @@ public class DefaultMQProducerTest { } }); countDownLatch.await(3000L, TimeUnit.MILLISECONDS); - assertThat(cc.get()).isEqualTo(1); } @Test From 1472dd80257c17efd5e5353a460b2f9ff25d0dae Mon Sep 17 00:00:00 2001 From: ShannonDing Date: Tue, 19 Nov 2019 20:49:12 +0800 Subject: [PATCH 68/73] feat(release): update client version to release 4.6.0 --- common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java index 47cfb6ebd0..019afd8c7f 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java @@ -18,7 +18,7 @@ package org.apache.rocketmq.common; public class MQVersion { - public static final int CURRENT_VERSION = Version.V4_5_2.ordinal(); + public static final int CURRENT_VERSION = Version.V4_6_0.ordinal(); public static String getVersionDesc(int value) { int length = Version.values().length; From d1b4e47c26576cf4fa543bbdd8c6f09643220bed Mon Sep 17 00:00:00 2001 From: ShannonDing Date: Tue, 19 Nov 2019 20:59:56 +0800 Subject: [PATCH 69/73] [maven-release-plugin] prepare release rocketmq-all-4.6.0 --- acl/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 6 +++--- filter/pom.xml | 2 +- logappender/pom.xml | 2 +- logging/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 9 ++++----- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 17 files changed, 22 insertions(+), 23 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index ec5ab0493f..98cb33f990 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 rocketmq-acl rocketmq-acl ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index 3852baa00e..f10e1e2313 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index c2ec6fab75..50d0958ce2 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 11df833bff..498d6c1bd5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index 4e3e78fcd9..2661d54a2b 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index 5e160276e5..f5fee205b0 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 @@ -51,12 +51,12 @@ org.apache.rocketmq rocketmq-openmessaging - 4.6.0-SNAPSHOT + 4.6.0 org.apache.rocketmq rocketmq-acl - 4.6.0-SNAPSHOT + 4.6.0 diff --git a/filter/pom.xml b/filter/pom.xml index 9b2128d957..5d92104cec 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index 667f213587..7ff3af9c5b 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index 95c70b504f..bf76b844e5 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index bb858b91e1..cc2a1b14cb 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index 8215a3f628..ac8ac82e75 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/pom.xml b/pom.xml index 14b10fae72..c4dcfc80ce 100644 --- a/pom.xml +++ b/pom.xml @@ -16,8 +16,7 @@ limitations under the License. --> - + org.apache @@ -30,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -43,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - HEAD + rocketmq-all-4.6.0 @@ -160,7 +159,7 @@ - + true diff --git a/remoting/pom.xml b/remoting/pom.xml index 68aebd9484..08785a0ef1 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 6aa9fa31a2..811a20ceb5 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index 79e337b0b8..a874b09445 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index c3d1318884..63a212821b 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 061f3c76bd..29aedb2e39 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0-SNAPSHOT + 4.6.0 4.0.0 From c3b7d18603d7d599a336703e871b86607958fc92 Mon Sep 17 00:00:00 2001 From: ShannonDing Date: Tue, 19 Nov 2019 21:00:19 +0800 Subject: [PATCH 70/73] [maven-release-plugin] prepare for next development iteration --- acl/pom.xml | 2 +- broker/pom.xml | 2 +- client/pom.xml | 2 +- common/pom.xml | 2 +- distribution/pom.xml | 2 +- example/pom.xml | 6 +++--- filter/pom.xml | 2 +- logappender/pom.xml | 2 +- logging/pom.xml | 2 +- namesrv/pom.xml | 2 +- openmessaging/pom.xml | 2 +- pom.xml | 4 ++-- remoting/pom.xml | 2 +- srvutil/pom.xml | 2 +- store/pom.xml | 2 +- test/pom.xml | 2 +- tools/pom.xml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/acl/pom.xml b/acl/pom.xml index 98cb33f990..6768ee29df 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT rocketmq-acl rocketmq-acl ${project.version} diff --git a/broker/pom.xml b/broker/pom.xml index f10e1e2313..5489ded026 100644 --- a/broker/pom.xml +++ b/broker/pom.xml @@ -13,7 +13,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/client/pom.xml b/client/pom.xml index 50d0958ce2..1cf292be5b 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/common/pom.xml b/common/pom.xml index 498d6c1bd5..b5a566dc18 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/distribution/pom.xml b/distribution/pom.xml index 2661d54a2b..c023c9847e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -20,7 +20,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT rocketmq-distribution rocketmq-distribution ${project.version} diff --git a/example/pom.xml b/example/pom.xml index f5fee205b0..8340d82642 100644 --- a/example/pom.xml +++ b/example/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 @@ -51,12 +51,12 @@ org.apache.rocketmq rocketmq-openmessaging - 4.6.0 + 4.6.1-SNAPSHOT org.apache.rocketmq rocketmq-acl - 4.6.0 + 4.6.1-SNAPSHOT diff --git a/filter/pom.xml b/filter/pom.xml index 5d92104cec..e170f26884 100644 --- a/filter/pom.xml +++ b/filter/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/logappender/pom.xml b/logappender/pom.xml index 7ff3af9c5b..83cd3957f7 100644 --- a/logappender/pom.xml +++ b/logappender/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 rocketmq-logappender diff --git a/logging/pom.xml b/logging/pom.xml index bf76b844e5..4b92a6ac7b 100644 --- a/logging/pom.xml +++ b/logging/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/namesrv/pom.xml b/namesrv/pom.xml index cc2a1b14cb..3b5fc5635e 100644 --- a/namesrv/pom.xml +++ b/namesrv/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml index ac8ac82e75..4a9354f1a3 100644 --- a/openmessaging/pom.xml +++ b/openmessaging/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index c4dcfc80ce..f8a391b20e 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2012 org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT pom Apache RocketMQ ${project.version} http://rocketmq.apache.org/ @@ -42,7 +42,7 @@ git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git scm:git:git@github.com:apache/rocketmq.git - rocketmq-all-4.6.0 + HEAD diff --git a/remoting/pom.xml b/remoting/pom.xml index 08785a0ef1..3443302f08 100644 --- a/remoting/pom.xml +++ b/remoting/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/srvutil/pom.xml b/srvutil/pom.xml index 811a20ceb5..5f4c4616ef 100644 --- a/srvutil/pom.xml +++ b/srvutil/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/store/pom.xml b/store/pom.xml index a874b09445..8aa9a84f59 100644 --- a/store/pom.xml +++ b/store/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/test/pom.xml b/test/pom.xml index 63a212821b..b7f8933e32 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -20,7 +20,7 @@ rocketmq-all org.apache.rocketmq - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 diff --git a/tools/pom.xml b/tools/pom.xml index 29aedb2e39..1d099eea5a 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -19,7 +19,7 @@ org.apache.rocketmq rocketmq-all - 4.6.0 + 4.6.1-SNAPSHOT 4.0.0 From a56bec8163dd16aa32c498b3b6a7100c612d91c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=8A=E5=90=8D?= Date: Mon, 25 Nov 2019 16:14:31 +0800 Subject: [PATCH 71/73] feat(PullConsumer) support begin/end seek support for pull consumer --- .../consumer/DefaultLitePullConsumer.java | 28 +++++++++++---- .../client/consumer/LitePullConsumer.java | 23 +++++++++++++ .../consumer/DefaultLitePullConsumerImpl.java | 34 +++++++++++++------ 3 files changed, 68 insertions(+), 17 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 99976d55d3..09f7e20455 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.client.consumer; import java.util.Collection; import java.util.List; - import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; import org.apache.rocketmq.client.consumer.store.OffsetStore; @@ -35,12 +34,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** - * Consumers belonging to the same consumer group share a group id. The consumers in a group then - * divides the topic as fairly amongst themselves as possible by establishing that each queue is only - * consumed by a single consumer from the group. If all consumers are from the same group, it functions - * as a traditional message queue. Each message would be consumed by one consumer of the group only. - * When multiple consumer groups exist, the flow of the data consumption model aligns with the traditional - * publish-subscribe model. The messages are broadcast to all consumer groups. + * Consumers belonging to the same consumer group share a group id. The consumers in a group then divides the topic + * as fairly amongst themselves as possible by establishing that each queue is only consumed by a single consumer + * from the group. If all consumers are from the same group, it functions as a traditional message queue. Each + * message would be consumed by one consumer of the group only. When multiple consumer groups exist, the flow of the + * data consumption model aligns with the traditional publish-subscribe model. The messages are broadcast to all + * consumer groups. */ private String consumerGroup; @@ -266,6 +265,21 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon return this.defaultLitePullConsumerImpl.committed(messageQueue); } + @Override + public void updateNameServerAddress(String nameServerAddress) { + this.defaultLitePullConsumerImpl.updateNameServerAddr(nameServerAddress); + } + + @Override + public void seekToBegin(MessageQueue messageQueue) throws MQClientException { + this.defaultLitePullConsumerImpl.seekToBegin(messageQueue); + } + + @Override + public void seekToEnd(MessageQueue messageQueue) throws MQClientException { + this.defaultLitePullConsumerImpl.seekToEnd(messageQueue); + } + @Override public boolean isAutoCommit() { return autoCommit; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java index d6e657ff18..ce22288033 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -172,4 +172,27 @@ public interface LitePullConsumer { */ void registerTopicMessageQueueChangeListener(String topic, TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException; + + /** + * Update name server addresses. + */ + void updateNameServerAddress(String nameServerAddress); + + /** + * Overrides the fetch offsets with the begin offset that the consumer will use on the next poll. If this API is + * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + */ + void seekToBegin(MessageQueue messageQueue)throws MQClientException; + + /** + * Overrides the fetch offsets with the end offset that the consumer will use on the next poll. If this API is + * invoked for the same message queue more than once, the latest offset will be used on the next poll(). Note that + * you may lose data if this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + */ + void seekToEnd(MessageQueue messageQueue)throws MQClientException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 2c673a1b06..f44eea7f9a 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -16,16 +16,16 @@ */ package org.apache.rocketmq.client.impl.consumer; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -35,13 +35,12 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; - import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; -import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; -import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; @@ -164,6 +163,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); } + public void updateNameServerAddr(String newAddresses) { + this.mQClientFactory.getMQClientAPIImpl().updateNameServerAddressList(newAddresses); + } + private synchronized void setSubscriptionType(SubscriptionType type) { if (this.subscriptionType == SubscriptionType.NONE) this.subscriptionType = type; @@ -556,6 +559,16 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } + public void seekToBegin(MessageQueue messageQueue) throws MQClientException { + long begin = minOffset(messageQueue); + this.seek(messageQueue, begin); + } + + public void seekToEnd(MessageQueue messageQueue) throws MQClientException { + long begin = maxOffset(messageQueue); + this.seek(messageQueue, begin); + } + private long maxOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); return this.mQClientFactory.getMQAdminImpl().maxOffset(messageQueue); @@ -764,8 +777,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), topic, SubscriptionData.SUB_ALL); } - PullResult pullResult = pull(messageQueue, subscriptionData, offset, nextPullBatchSize()); + + switch (pullResult.getPullStatus()) { case FOUND: final Object objLock = messageQueueLock.fetchLockObject(messageQueue); From 0bddfe50747c270e6c2312a75ff695361e583225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BF=8A=E5=90=8D?= Date: Mon, 25 Nov 2019 17:28:03 +0800 Subject: [PATCH 72/73] test(PullConsumer) add seek to beigin/end unit test --- .../consumer/DefaultLitePullConsumerTest.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index b55f8a0a1c..d2cb0571ed 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -189,6 +189,36 @@ public class DefaultLitePullConsumerTest { litePullConsumer.shutdown(); } + @Test + public void testSeek_SeekToBegin() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.seekToBegin(messageQueue); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 0L); + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_SeekToEnd() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.seekToEnd(messageQueue); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 500L); + litePullConsumer.shutdown(); + } + @Test public void testSeek_SeekOffsetIllegal() throws Exception { DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); From 44569e4df6620fff61e43d782815f93a97e748ea Mon Sep 17 00:00:00 2001 From: duhenglucky Date: Tue, 26 Nov 2019 11:28:33 +0800 Subject: [PATCH 73/73] fix(PullConsumer) fix the namespace issue of pull consumer's consumerGroup --- .../rocketmq/client/consumer/DefaultLitePullConsumer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 09f7e20455..52060ea753 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -26,6 +26,7 @@ import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; @@ -186,6 +187,7 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void start() throws MQClientException { + setConsumerGroup(NamespaceUtil.wrapNamespace(this.getNamespace(), this.consumerGroup)); this.defaultLitePullConsumerImpl.start(); } @@ -425,4 +427,8 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) { this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis; } + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } }