From 2aae40f9256cb0a6a2b628f00df7dbe9718a963d Mon Sep 17 00:00:00 2001 From: duhenglucky Date: Thu, 20 Jun 2019 23:32:33 +0800 Subject: [PATCH 01/14] Add pull schedual service --- .../MQPullConsumerScheduleService.java | 2 +- .../impl/consumer/AssignedMessageQueue.java | 157 +++++++++++ .../consumer/DefaultMQPullConsumerImpl.java | 6 +- .../impl/consumer/LiteMQPullConsumerImpl.java | 255 ++++++++++++++++++ 4 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java index 44b864e348..685f4c8d2d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -151,7 +151,7 @@ public class MQPullConsumerScheduleService { } } - class PullTaskImpl implements Runnable { + public class PullTaskImpl implements Runnable { private final MessageQueue messageQueue; private volatile boolean cancelled = false; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java new file mode 100644 index 0000000000..e9623a8157 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageQueue; + +public class AssignedMessageQueue { + + private ConcurrentHashMap assignedMessageQueueState; + + public AssignedMessageQueue() { + assignedMessageQueueState = new ConcurrentHashMap(); + } + + public boolean isPaused(MessageQueue messageQueue) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + return messageQueueStat.isPaused(); + } + return false; + } + + public void pause(Collection messageQueues) { + for (MessageQueue messageQueue : messageQueues) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (assignedMessageQueueState.get(messageQueue) != null) { + messageQueueStat.setPaused(true); + } + } + } + + public void resume(Collection messageQueueCollection) { + for (MessageQueue messageQueue : messageQueueCollection) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (assignedMessageQueueState.get(messageQueue) != null) { + messageQueueStat.setPaused(false); + } + } + } + + public long getNextOffset(MessageQueue messageQueue) throws MQClientException { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (assignedMessageQueueState.get(messageQueue) != null) { + return messageQueueStat.getNextOffset(); + } + return -1; + } + + public void updateNextOffset(MessageQueue messageQueue, long offset) throws MQClientException { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueue == null) { + messageQueueStat = new MessageQueueStat(messageQueue, offset); + assignedMessageQueueState.putIfAbsent(messageQueue, messageQueueStat); + } + assignedMessageQueueState.get(messageQueue).setNextOffset(offset); + } + + public void updateAssignedMessageQueue(Set assigned) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (!assigned.contains(next.getKey())) { + it.remove(); + } + } + + for (MessageQueue messageQueue : assigned) { + if (!this.assignedMessageQueueState.containsKey(messageQueue)) { + MessageQueueStat messageQueueStat = new MessageQueueStat(messageQueue); + this.assignedMessageQueueState.put(messageQueue, messageQueueStat); + } + } + } + } + + public void removeAssignedMessageQueue(String topic) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + it.remove(); + } + } + } + } + + public Map getNeedCommitOffsets() { + Map map = new HashMap(); + Set> entries = this.assignedMessageQueueState.entrySet(); + for (Map.Entry entry : entries) { + map.put(entry.getKey(), entry.getValue().getNextOffset()); + } + return map; + } + + public class MessageQueueStat { + private MessageQueue messageQueue; + private boolean paused = false; + private long nextOffset = -1; + + public MessageQueueStat(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + public MessageQueueStat(MessageQueue messageQueue, long nextOffset) { + this.messageQueue = messageQueue; + this.nextOffset = nextOffset; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + public boolean isPaused() { + return paused; + } + + public void setPaused(boolean paused) { + this.paused = paused; + } + + public long getNextOffset() { + return nextOffset; + } + + public void setNextOffset(long nextOffset) { + this.nextOffset = nextOffset; + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 8aff14b748..bc0884a4ef 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -68,16 +68,16 @@ import org.apache.rocketmq.remoting.exception.RemotingException; public class DefaultMQPullConsumerImpl implements MQConsumerInner { private final InternalLogger log = ClientLogger.getLog(); - private final DefaultMQPullConsumer defaultMQPullConsumer; + protected final DefaultMQPullConsumer defaultMQPullConsumer; private final long consumerStartTimestamp = System.currentTimeMillis(); private final RPCHook rpcHook; private final ArrayList consumeMessageHookList = new ArrayList(); private final ArrayList filterMessageHookList = new ArrayList(); private volatile ServiceState serviceState = ServiceState.CREATE_JUST; - private MQClientInstance mQClientFactory; + protected MQClientInstance mQClientFactory; private PullAPIWrapper pullAPIWrapper; private OffsetStore offsetStore; - private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); + protected RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); public DefaultMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { this.defaultMQPullConsumer = defaultMQPullConsumer; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java new file mode 100644 index 0000000000..73328180d9 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.RPCHook; + +public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { + private final InternalLogger log = ClientLogger.getLog(); + + private final ConcurrentMap taskTable = + new ConcurrentHashMap(); + + private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue(); + + private List allConsumed = new ArrayList(256); + + private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); + ; + + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + + public LiteMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { + super(defaultMQPullConsumer, rpcHook); + } + + public void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { + this.assignedMessageQueue.updateAssignedMessageQueue(assignedMessageQueue); + updatePullTask(topic, assignedMessageQueue); + } + + public void updatePullTask(String topic, Set mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + + for (MessageQueue messageQueue : mqNewSet) { + if (!this.taskTable.containsKey(messageQueue)) { + PullTaskImpl pullTask = new PullTaskImpl(messageQueue); + this.taskTable.put(messageQueue, pullTask); + this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS); + } + } + } + + class MessageQueueListenerImpl implements MessageQueueListener { + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageModel messageModel = defaultMQPullConsumer.getMessageModel(); + switch (messageModel) { + case BROADCASTING: + updateAssignedMessageQueue(topic, mqAll); + break; + case CLUSTERING: + updateAssignedMessageQueue(topic, mqDivided); + break; + default: + break; + } + } + } + + int nextPullBatchNums() { + return Math.min(10, consumeRequestCache.remainingCapacity()); + } + + @Override + public synchronized void start() throws MQClientException { + super.start(); + final String group = this.defaultMQPullConsumer.getConsumerGroup(); + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + 10, //this.pullThreadNums, + new ThreadFactoryImpl("PullMsgThread-" + group) + ); + this.defaultMQPullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + } + + public void subscribe(String topic, String subExpression) throws MQClientException { + try { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultMQPullConsumer.getConsumerGroup(), + topic, subExpression); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + if (this.mQClientFactory != null) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { + try { + assignedMessageQueue.updateNextOffset(remoteQueue, nextPullOffset); + } catch (MQClientException e) { + log.error("A error occurred in update consume: {} offset process.", remoteQueue, e); + } + } + + private void addToConsumed(ConsumeRequest consumeRequest) { + synchronized (this.allConsumed) { + allConsumed.add(consumeRequest); + } + } + + void submitConsumeRequest(ConsumeRequest consumeRequest) { + try { + consumeRequestCache.put(consumeRequest); + addToConsumed(consumeRequest); + } catch (InterruptedException ex) { + log.error("Submit consumeRequest error", ex); + } + } + + long nextPullOffset(MessageQueue remoteQueue) { + long offset = -1; + try { + offset = assignedMessageQueue.getNextOffset(remoteQueue); + if (offset == -1) { + offset = this.defaultMQPullConsumer.fetchConsumeOffset(remoteQueue, false); + assignedMessageQueue.updateNextOffset(remoteQueue, offset); + } + } catch (MQClientException e) { + log.error("An error occurred in fetch consume offset process.", e); + } + return offset; + } + + public class PullTaskImpl implements Runnable { + private final MessageQueue messageQueue; + private volatile boolean cancelled = false; + + public PullTaskImpl(final MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + @Override + public void run() { + String topic = this.messageQueue.getTopic(); + if (!this.isCancelled()) { + if (assignedMessageQueue.isPaused(messageQueue)) { + log.debug("Message Queue: {} has been paused!", messageQueue); + return; + } + SubscriptionData subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); + long offset = nextPullOffset(messageQueue); + try { + PullResult pullResult = defaultMQPullConsumer.pull(messageQueue, subscriptionData.getSubString(), offset, nextPullBatchNums()); + ProcessQueue processQueue = rebalanceImpl.getProcessQueueTable().get(messageQueue); + switch (pullResult.getPullStatus()) { + case FOUND: + if (processQueue != null) { + processQueue.putMessage(pullResult.getMsgFoundList()); + submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + } + break; + default: + break; + } + updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); + } catch (Exception e) { + log.error("An error occurred in pull message process.", e); + } + } + } + + public boolean isCancelled() { + return cancelled; + } + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + public class ConsumeRequest { + private final List messageExts; + private final MessageQueue messageQueue; + private final ProcessQueue processQueue; + private long startConsumeTimeMillis; + + public ConsumeRequest(final List messageExts, final MessageQueue messageQueue, + final ProcessQueue processQueue) { + this.messageExts = messageExts; + this.messageQueue = messageQueue; + this.processQueue = processQueue; + } + + public List getMessageExts() { + return messageExts; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + public long getStartConsumeTimeMillis() { + return startConsumeTimeMillis; + } + + public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) { + this.startConsumeTimeMillis = startConsumeTimeMillis; + } + } +} From af04557abf12ce36245d7e362925125cd25e0d3c Mon Sep 17 00:00:00 2001 From: duhenglucky Date: Fri, 21 Jun 2019 09:47:42 +0800 Subject: [PATCH 02/14] Add pull task logic --- .../consumer/DefaultLiteMQPullConsumer.java | 64 ++++++++ .../consumer/DefaultMQPullConsumer.java | 2 + .../client/consumer/LiteMQPullConsumer.java | 55 +++++++ .../client/consumer/MQPullConsumer.java | 3 + .../impl/consumer/LiteMQPullConsumerImpl.java | 140 +++++++++++++++++- 5 files changed, 263 insertions(+), 1 deletion(-) create mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java new file mode 100644 index 0000000000..99fd0d9768 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Collection; +import java.util.List; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.consumer.LiteMQPullConsumerImpl; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.remoting.RPCHook; + +public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements LiteMQPullConsumer { + private LiteMQPullConsumerImpl liteMQPullConsumer; + + public DefaultLiteMQPullConsumer(String consumerGroup, RPCHook rpcHook) { + this.liteMQPullConsumer = new LiteMQPullConsumerImpl(this, rpcHook); + } + + @Override public void subscribe(String topic, String subExpression) throws MQClientException{ + this.liteMQPullConsumer.subscribe(topic, subExpression); + } + + @Override public void unsubscribe(String topic) { + } + + @Override public List poll() { + return poll(this.getConsumerPullTimeoutMillis()); + } + + @Override public List poll(long timeout) { + return liteMQPullConsumer.poll(timeout); + } + + @Override public void seek(MessageQueue messageQueue, long offset) throws MQClientException { + + } + + @Override public void pause(Collection messageQueueCollection) { + + } + + @Override public void resume(Collection partitions) { + + } + + @Override public void commitSync() { + + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index f3b6caaa71..dbf37d2a2d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -16,7 +16,9 @@ */ package org.apache.rocketmq.client.consumer; +import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java new file mode 100644 index 0000000000..223cca076e --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Collection; +import java.util.List; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; + +public interface LiteMQPullConsumer { + /** + * Subscribe some topic + * + * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if + * null or * expression,meaning subscribe all + */ + void subscribe(final String topic, final String subExpression) throws MQClientException; + + /** + * Unsubscribe consumption some topic + * + * @param topic message topic + */ + void unsubscribe(final String topic); + + /** + * @return + */ + List poll(); + + List poll(long timeout); + + void seek(MessageQueue messageQueue, long offset) throws MQClientException; + + void pause(Collection messageQueueCollection); + + void resume(Collection partitions); + + void commitSync(); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java index 28b807c2ed..9c7cb363f2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java @@ -16,6 +16,8 @@ */ package org.apache.rocketmq.client.consumer; +import java.util.Collection; +import java.util.List; import java.util.Set; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; @@ -169,4 +171,5 @@ public interface MQPullConsumer extends MQConsumer { */ void sendMessageBack(MessageExt msg, int delayLevel, String brokerName, String consumerGroup) throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java index 73328180d9..abf5f47476 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java @@ -17,16 +17,21 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.PullResult; @@ -34,6 +39,7 @@ import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; @@ -118,6 +124,81 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { this.defaultMQPullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); } + public List poll(long timeout) { + try { + ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS); + if (consumeRequest != null) { + List messages = consumeRequest.getMessageExts(); + for (MessageExt messageExt : messages) { + MessageAccessor.setConsumeStartTimeStamp(messageExt, String.valueOf(consumeRequest.getStartConsumeTimeMillis())); + } + consumeRequest.setStartConsumeTimeMillis(System.currentTimeMillis()); + return messages; + } + } catch (InterruptedException ignore) { + } + return null; + } + + public void pause(Collection messageQueues) { + assignedMessageQueue.pause(messageQueues); + } + + public void resume(Collection messageQueues) { + assignedMessageQueue.resume(messageQueues); + } + + public void unsubscribe(final String topic) { + unsubscribe(topic); + removePullTaskCallback(topic); + assignedMessageQueue.removeAssignedMessageQueue(topic); + } + + public void removePullTaskCallback(final String topic) { + removePullTask(topic); + } + + public void removePullTask(final String topic) { + synchronized (this.taskTable) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + it.remove(); + } + } + } + } + + public void commit() { + List consumeRequests; + synchronized (this.allConsumed) { + consumeRequests = this.allConsumed; + this.allConsumed = new ArrayList(); + } + for (ConsumeRequest consumeRequest : consumeRequests) { + consumeRequest.getProcessQueue().removeMessage(consumeRequest.messageExts); + } + Set> entrySet = assignedMessageQueue.getNeedCommitOffsets().entrySet(); + for (Map.Entry entry : entrySet) { + try { + updateConsumeOffset(entry.getKey(), entry.getValue()); + } catch (MQClientException e) { + log.error("A error occurred in update consume offset process.", e); + } + } + this.getOffsetStore().persistAll(assignedMessageQueue.getNeedCommitOffsets().keySet()); + } + + private void commit(final MessageQueue messageQueue, final ProcessQueue processQueue, final MessageExt messageExt) { + long offset = processQueue.removeMessage(Collections.singletonList(messageExt)); + try { + updateConsumeOffset(messageQueue, offset); + } catch (MQClientException e) { + log.error("An error occurred in update consume offset process.", e); + } + } + public void subscribe(String topic, String subExpression) throws MQClientException { try { SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultMQPullConsumer.getConsumerGroup(), @@ -159,7 +240,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { try { offset = assignedMessageQueue.getNextOffset(remoteQueue); if (offset == -1) { - offset = this.defaultMQPullConsumer.fetchConsumeOffset(remoteQueue, false); + offset = fetchConsumeOffset(remoteQueue, false); assignedMessageQueue.updateNextOffset(remoteQueue, offset); } } catch (MQClientException e) { @@ -168,6 +249,63 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { return offset; } + private void cleanExpireMsg() { + for (final Map.Entry next : rebalanceImpl.getProcessQueueTable().entrySet()) { + ProcessQueue pq = next.getValue(); + MessageQueue mq = next.getKey(); + ReadWriteLock lockTreeMap = getLockInProcessQueue(pq); + if (lockTreeMap == null) { + log.error("Gets tree map lock in process queue error of message queue:", mq); + return; + } + + TreeMap msgTreeMap = pq.getMsgTreeMap(); + + int loop = msgTreeMap.size(); + for (int i = 0; i < loop; i++) { + MessageExt msg = null; + try { + lockTreeMap.readLock().lockInterruptibly(); + try { + if (!msgTreeMap.isEmpty()) { + msg = msgTreeMap.firstEntry().getValue(); + if (System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msg)) + > 10 * 60 * 1000) { + //Expired, ack and remove it. + } else { + break; + } + } else { + break; + } + } finally { + lockTreeMap.readLock().unlock(); + } + } catch (InterruptedException e) { + log.error("Gets expired message exception", e); + } + + try { + this.defaultMQPullConsumer.sendMessageBack(msg, 3); + log.info("Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", + msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); + System.out.println("Send expired msg back."); + commit(mq, pq, msg); + } catch (Exception e) { + log.error("Send back expired msg exception", e); + } + } + } + } + + private ReadWriteLock getLockInProcessQueue(ProcessQueue pq) { + try { + return (ReadWriteLock) FieldUtils.readDeclaredField(pq, "lockTreeMap", true); + } catch (IllegalAccessException e) { + return null; + } + } + public class PullTaskImpl implements Runnable { private final MessageQueue messageQueue; private volatile boolean cancelled = false; From 11b686e47797be15f187f906e4064264dfa2c993 Mon Sep 17 00:00:00 2001 From: duhenglucky Date: Wed, 17 Jul 2019 14:33:29 +0800 Subject: [PATCH 03/14] Add lite pull consumer example --- .../consumer/DefaultLiteMQPullConsumer.java | 88 ++++++++++++++++--- .../client/consumer/LiteMQPullConsumer.java | 3 - .../impl/consumer/LiteMQPullConsumerImpl.java | 77 ++++++++++++++-- .../example/simple/LitePullConsumerTest.java | 49 +++++++++++ 4 files changed, 195 insertions(+), 22 deletions(-) create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java index 99fd0d9768..96d4f5abe3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java @@ -27,18 +27,50 @@ import org.apache.rocketmq.remoting.RPCHook; public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements LiteMQPullConsumer { private LiteMQPullConsumerImpl liteMQPullConsumer; + /** + * Maximum amount of time in minutes a message may block the consuming thread. + */ + private long consumeTimeout = 15; + + /** + * Is auto commit offset + */ + private boolean autoCommit = true; + + private int pullThreadNumbers = 20; + + /** + * Maximum commit offset interval time in seconds. + */ + private long autoCommitInterval = 20; + public DefaultLiteMQPullConsumer(String consumerGroup, RPCHook rpcHook) { + this.setConsumerGroup(consumerGroup); this.liteMQPullConsumer = new LiteMQPullConsumerImpl(this, rpcHook); } - @Override public void subscribe(String topic, String subExpression) throws MQClientException{ + public DefaultLiteMQPullConsumer(String consumerGroup) { + this.setConsumerGroup(consumerGroup); + this.liteMQPullConsumer = new LiteMQPullConsumerImpl(this, null); + } + + @Override + public void start() throws MQClientException{ + this.liteMQPullConsumer.start(); + } + + @Override + public void subscribe(String topic, String subExpression) throws MQClientException { this.liteMQPullConsumer.subscribe(topic, subExpression); } - @Override public void unsubscribe(String topic) { + @Override + public void unsubscribe(String topic) { + this.liteMQPullConsumer.unsubscribe(topic); } - @Override public List poll() { + @Override + public List poll() { return poll(this.getConsumerPullTimeoutMillis()); } @@ -46,19 +78,55 @@ public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements return liteMQPullConsumer.poll(timeout); } - @Override public void seek(MessageQueue messageQueue, long offset) throws MQClientException { - + @Override + public void seek(MessageQueue messageQueue, long offset) throws MQClientException { + this.liteMQPullConsumer.seek(messageQueue, offset); } - @Override public void pause(Collection messageQueueCollection) { - + @Override + public void pause(Collection messageQueues) { + this.liteMQPullConsumer.pause(messageQueues); } - @Override public void resume(Collection partitions) { - + @Override + public void resume(Collection messageQueues) { + this.liteMQPullConsumer.resume(messageQueues); } - @Override public void commitSync() { + @Override + public void commitSync() { + this.liteMQPullConsumer.commit(); + } + public long getConsumeTimeout() { + return consumeTimeout; + } + + public void setConsumeTimeout(long consumeTimeout) { + this.consumeTimeout = consumeTimeout; + } + + public boolean isAutoCommit() { + return autoCommit; + } + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + public int getPullThreadNumbers() { + return pullThreadNumbers; + } + + public void setPullThreadNumbers(int pullThreadNumbers) { + this.pullThreadNumbers = pullThreadNumbers; + } + + public long getAutoCommitInterval() { + return autoCommitInterval; + } + + public void setAutoCommitInterval(long autoCommitInterval) { + this.autoCommitInterval = autoCommitInterval; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java index 223cca076e..da8d1cf1d7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java @@ -38,9 +38,6 @@ public interface LiteMQPullConsumer { */ void unsubscribe(final String topic); - /** - * @return - */ List poll(); List poll(long timeout); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java index abf5f47476..d612286276 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java @@ -27,12 +27,14 @@ import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.DefaultLiteMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.exception.MQClientException; @@ -50,6 +52,8 @@ import org.apache.rocketmq.remoting.RPCHook; public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { private final InternalLogger log = ClientLogger.getLog(); + private DefaultLiteMQPullConsumer defaultLiteMQPullConsumer; + private final ConcurrentMap taskTable = new ConcurrentHashMap(); @@ -58,12 +62,21 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { private List allConsumed = new ArrayList(256); private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); - ; + + private final ScheduledExecutorService cleanExpireMsgExecutors; private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; - public LiteMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { + private ScheduledExecutorService autoCommitExecutors; + + public LiteMQPullConsumerImpl(final DefaultLiteMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { super(defaultMQPullConsumer, rpcHook); + this.defaultLiteMQPullConsumer = defaultMQPullConsumer; + this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "Lite_CleanExpireMsgScheduledThread_")); + this.autoCommitExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "Lite_AutoCommitScheduledThread_")); + } public void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { @@ -115,18 +128,43 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { @Override public synchronized void start() throws MQClientException { + this.defaultMQPullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); super.start(); final String group = this.defaultMQPullConsumer.getConsumerGroup(); this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - 10, //this.pullThreadNums, + this.defaultLiteMQPullConsumer.getPullThreadNumbers(), new ThreadFactoryImpl("PullMsgThread-" + group) ); - this.defaultMQPullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + cleanExpireMsg(); + } + }, this.defaultLiteMQPullConsumer.getConsumeTimeout(), this.defaultLiteMQPullConsumer.getConsumeTimeout(), TimeUnit.MINUTES); + this.autoCommitExecutors.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + if (defaultLiteMQPullConsumer.isAutoCommit()) { + commit(); + } + } + }, this.defaultLiteMQPullConsumer.getAutoCommitInterval(), this.defaultLiteMQPullConsumer.getAutoCommitInterval(), TimeUnit.SECONDS); + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + + private void updateTopicSubscribeInfoWhenSubscriptionChanged() { + Map subTable = rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + } } public List poll(long timeout) { try { - ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS); + ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.SECONDS); if (consumeRequest != null) { List messages = consumeRequest.getMessageExts(); for (MessageExt messageExt : messages) { @@ -148,6 +186,16 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { assignedMessageQueue.resume(messageQueues); } + public void seek(MessageQueue messageQueue, long offset) throws MQClientException { + this.updatePullOffset(messageQueue, offset); + try { + updateConsumeOffset(messageQueue, offset); + } catch (MQClientException ex) { + log.error("Seek offset to remote message queue error!", ex); + throw ex; + } + } + public void unsubscribe(final String topic) { unsubscribe(topic); removePullTaskCallback(topic); @@ -270,7 +318,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { if (!msgTreeMap.isEmpty()) { msg = msgTreeMap.firstEntry().getValue(); if (System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msg)) - > 10 * 60 * 1000) { + > this.defaultLiteMQPullConsumer.getConsumeTimeout() * 60 * 1000) { //Expired, ack and remove it. } else { break; @@ -316,16 +364,19 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { @Override public void run() { + System.out.println("begin pull message"); String topic = this.messageQueue.getTopic(); if (!this.isCancelled()) { if (assignedMessageQueue.isPaused(messageQueue)) { + scheduledThreadPoolExecutor.schedule(this, 1000, TimeUnit.MILLISECONDS); log.debug("Message Queue: {} has been paused!", messageQueue); return; } SubscriptionData subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); long offset = nextPullOffset(messageQueue); + long pullDelayTimeMills = 0; try { - PullResult pullResult = defaultMQPullConsumer.pull(messageQueue, subscriptionData.getSubString(), offset, nextPullBatchNums()); + PullResult pullResult = pull(messageQueue, subscriptionData.getSubString(), offset, nextPullBatchNums()); ProcessQueue processQueue = rebalanceImpl.getProcessQueueTable().get(messageQueue); switch (pullResult.getPullStatus()) { case FOUND: @@ -338,9 +389,17 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { break; } updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); - } catch (Exception e) { + } catch (Throwable e) { + pullDelayTimeMills = 1000; + e.printStackTrace(); log.error("An error occurred in pull message process.", e); } + + if (!this.isCancelled()) { + scheduledThreadPoolExecutor.schedule(this, pullDelayTimeMills, TimeUnit.MILLISECONDS); + } else { + log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); + } } } diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java new file mode 100644 index 0000000000..4297e4f16e --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.Arrays; +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultLiteMQPullConsumer; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; + +public class LitePullConsumerTest { + public static void main(String[] args) throws Exception { + DefaultLiteMQPullConsumer litePullConsumer = new DefaultLiteMQPullConsumer("test", null); + litePullConsumer.subscribe("test", null); + litePullConsumer.start(); + MessageQueue messageQueue = new MessageQueue("test", "duhengdeMacBook-Pro.local", 1); + int i = 0; + while (true) { + List messageExts = litePullConsumer.poll(); + System.out.println("-----------"); + System.out.println(messageExts); + System.out.println("-----------"); + i++; + if (i == 3) { + System.out.println("pause"); + litePullConsumer.pause(Arrays.asList(messageQueue)); + } + if (i == 10) { + System.out.println("resume"); + litePullConsumer.resume(Arrays.asList(messageQueue)); + } + litePullConsumer.commitSync(); + } + } +} From 62ca947a457769469189e6e0acd990a833b70ab8 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Fri, 19 Jul 2019 11:19:07 +0800 Subject: [PATCH 04/14] Polish LitePullConsumer (#1332) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle --- .../consumer/DefaultLiteMQPullConsumer.java | 6 +- .../consumer/DefaultMQPullConsumer.java | 3 +- .../client/consumer/MQPullConsumer.java | 2 - .../impl/consumer/AssignedMessageQueue.java | 10 --- .../impl/consumer/LiteMQPullConsumerImpl.java | 65 ++++++++++++------- .../client/impl/consumer/ProcessQueue.java | 12 ++++ .../example/simple/LitePullConsumerTest.java | 14 ++-- 7 files changed, 64 insertions(+), 48 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java index 96d4f5abe3..6f67bcf88b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java @@ -42,7 +42,7 @@ public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements /** * Maximum commit offset interval time in seconds. */ - private long autoCommitInterval = 20; + private long autoCommitInterval = 5; public DefaultLiteMQPullConsumer(String consumerGroup, RPCHook rpcHook) { this.setConsumerGroup(consumerGroup); @@ -55,7 +55,7 @@ public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements } @Override - public void start() throws MQClientException{ + public void start() throws MQClientException { this.liteMQPullConsumer.start(); } @@ -95,7 +95,7 @@ public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements @Override public void commitSync() { - this.liteMQPullConsumer.commit(); + this.liteMQPullConsumer.commitSync(); } public long getConsumeTimeout() { diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index dbf37d2a2d..3fa3af2402 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -16,9 +16,7 @@ */ package org.apache.rocketmq.client.consumer; -import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.QueryResult; @@ -40,6 +38,7 @@ import org.apache.rocketmq.remoting.exception.RemotingException; * Default pulling consumer */ public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer { + protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java index 9c7cb363f2..a8e96283f9 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumer.java @@ -16,8 +16,6 @@ */ package org.apache.rocketmq.client.consumer; -import java.util.Collection; -import java.util.List; import java.util.Set; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index e9623a8157..fb0ca79b4e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -17,7 +17,6 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -107,15 +106,6 @@ public class AssignedMessageQueue { } } - public Map getNeedCommitOffsets() { - Map map = new HashMap(); - Set> entries = this.assignedMessageQueueState.entrySet(); - for (Map.Entry entry : entries) { - map.put(entry.getKey(), entry.getValue().getNextOffset()); - } - return map; - } - public class MessageQueueStat { private MessageQueue messageQueue; private boolean paused = false; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java index d612286276..ab229e444e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java @@ -16,9 +16,9 @@ */ package org.apache.rocketmq.client.impl.consumer; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -33,6 +33,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReadWriteLock; + import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.consumer.DefaultLiteMQPullConsumer; import org.apache.rocketmq.client.consumer.MessageQueueListener; @@ -50,6 +51,7 @@ import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { + private final InternalLogger log = ClientLogger.getLog(); private DefaultLiteMQPullConsumer defaultLiteMQPullConsumer; @@ -59,7 +61,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue(); - private List allConsumed = new ArrayList(256); + private volatile Set consumedSet = new HashSet(); private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); @@ -69,6 +71,8 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { private ScheduledExecutorService autoCommitExecutors; + private final ThreadLocal preConsumeRequestLocal = new ThreadLocal(); + public LiteMQPullConsumerImpl(final DefaultLiteMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { super(defaultMQPullConsumer, rpcHook); this.defaultLiteMQPullConsumer = defaultMQPullConsumer; @@ -145,7 +149,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { @Override public void run() { if (defaultLiteMQPullConsumer.isAutoCommit()) { - commit(); + commitAll(); } } }, this.defaultLiteMQPullConsumer.getAutoCommitInterval(), this.defaultLiteMQPullConsumer.getAutoCommitInterval(), TimeUnit.SECONDS); @@ -164,7 +168,9 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { public List poll(long timeout) { try { - ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.SECONDS); + addToConsumed(preConsumeRequestLocal.get()); + ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS); + preConsumeRequestLocal.set(consumeRequest); if (consumeRequest != null) { List messages = consumeRequest.getMessageExts(); for (MessageExt messageExt : messages) { @@ -173,7 +179,8 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { consumeRequest.setStartConsumeTimeMillis(System.currentTimeMillis()); return messages; } - } catch (InterruptedException ignore) { + } catch (InterruptedException e) { + log.error("poll ComsumeRequest error.", e); } return null; } @@ -197,7 +204,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { } public void unsubscribe(final String topic) { - unsubscribe(topic); + super.unsubscribe(topic); removePullTaskCallback(topic); assignedMessageQueue.removeAssignedMessageQueue(topic); } @@ -212,30 +219,39 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { while (it.hasNext()) { Map.Entry next = it.next(); if (next.getKey().getTopic().equals(topic)) { + next.getValue().setCancelled(true); it.remove(); } } } } - public void commit() { - List consumeRequests; - synchronized (this.allConsumed) { - consumeRequests = this.allConsumed; - this.allConsumed = new ArrayList(); + public void commitSync() { + addToConsumed(preConsumeRequestLocal.get()); + preConsumeRequestLocal.set(null); + commitAll(); + } + + public void commitAll() { + Set consumedRequests; + synchronized (this.consumedSet) { + consumedRequests = this.consumedSet; + this.consumedSet = new HashSet(); } - for (ConsumeRequest consumeRequest : consumeRequests) { + for (ConsumeRequest consumeRequest : consumedRequests) { consumeRequest.getProcessQueue().removeMessage(consumeRequest.messageExts); } - Set> entrySet = assignedMessageQueue.getNeedCommitOffsets().entrySet(); - for (Map.Entry entry : entrySet) { + Set> entrySet = this.rebalanceImpl.getProcessQueueTable().entrySet(); + for (Map.Entry entry : entrySet) { try { - updateConsumeOffset(entry.getKey(), entry.getValue()); + long consumeOffset = entry.getValue().getConsumeOffset(); + if (consumeOffset != -1) + updateConsumeOffset(entry.getKey(), consumeOffset); } catch (MQClientException e) { log.error("A error occurred in update consume offset process.", e); } } - this.getOffsetStore().persistAll(assignedMessageQueue.getNeedCommitOffsets().keySet()); + this.getOffsetStore().persistAll(this.rebalanceImpl.getProcessQueueTable().keySet()); } private void commit(final MessageQueue messageQueue, final ProcessQueue processQueue, final MessageExt messageExt) { @@ -260,7 +276,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { } } - void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { + private void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { try { assignedMessageQueue.updateNextOffset(remoteQueue, nextPullOffset); } catch (MQClientException e) { @@ -269,21 +285,23 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { } private void addToConsumed(ConsumeRequest consumeRequest) { - synchronized (this.allConsumed) { - allConsumed.add(consumeRequest); + if (consumeRequest != null) { + synchronized (this.consumedSet) { + if (!consumedSet.contains(consumeRequest)) + consumedSet.add(consumeRequest); + } } } - void submitConsumeRequest(ConsumeRequest consumeRequest) { + private void submitConsumeRequest(ConsumeRequest consumeRequest) { try { consumeRequestCache.put(consumeRequest); - addToConsumed(consumeRequest); } catch (InterruptedException ex) { log.error("Submit consumeRequest error", ex); } } - long nextPullOffset(MessageQueue remoteQueue) { + private long nextPullOffset(MessageQueue remoteQueue) { long offset = -1; try { offset = assignedMessageQueue.getNextOffset(remoteQueue); @@ -337,7 +355,7 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { this.defaultMQPullConsumer.sendMessageBack(msg, 3); log.info("Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); - System.out.println("Send expired msg back."); + log.info("Send expired msg back."); commit(mq, pq, msg); } catch (Exception e) { log.error("Send back expired msg exception", e); @@ -364,7 +382,6 @@ public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { @Override public void run() { - System.out.println("begin pull message"); String topic = this.messageQueue.getTopic(); if (!this.isCancelled()) { if (assignedMessageQueue.isPaused(messageQueue)) { diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index 0a52817ced..e9a1c72ddf 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -26,6 +26,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.logging.InternalLogger; @@ -431,4 +432,15 @@ public class ProcessQueue { public void setLastConsumeTimestamp(long lastConsumeTimestamp) { this.lastConsumeTimestamp = lastConsumeTimestamp; } + + public long getConsumeOffset() { + + if (msgTreeMap.isEmpty() && queueOffsetMax == 0L) + return -1; + + if (!msgTreeMap.isEmpty()) + return msgTreeMap.firstKey(); + else + return queueOffsetMax + 1; + } } diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java index 4297e4f16e..215763bd63 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java @@ -25,24 +25,24 @@ import org.apache.rocketmq.common.message.MessageQueue; public class LitePullConsumerTest { public static void main(String[] args) throws Exception { DefaultLiteMQPullConsumer litePullConsumer = new DefaultLiteMQPullConsumer("test", null); - litePullConsumer.subscribe("test", null); + litePullConsumer.setNamesrvAddr("localhost:9876"); + litePullConsumer.subscribe("litepullconsumertest9", null); litePullConsumer.start(); - MessageQueue messageQueue = new MessageQueue("test", "duhengdeMacBook-Pro.local", 1); + MessageQueue messageQueue = new MessageQueue("test", "IT-C02YW28FLVDL.local", 1); int i = 0; while (true) { List messageExts = litePullConsumer.poll(); - System.out.println("-----------"); - System.out.println(messageExts); - System.out.println("-----------"); + System.out.printf("%s%n", messageExts); i++; if (i == 3) { - System.out.println("pause"); + System.out.printf("pause%n"); litePullConsumer.pause(Arrays.asList(messageQueue)); } if (i == 10) { - System.out.println("resume"); + System.out.printf("resume%n"); litePullConsumer.resume(Arrays.asList(messageQueue)); } +// litePullConsumer.commitSync(); } } From 83098eb604007eca7892565df7c213ee19a342f5 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Tue, 30 Jul 2019 10:59:13 +0800 Subject: [PATCH 05/14] Polish lite pull consumer (#1349) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle * polish LiteMQPullConsumer * add flow control and polish commit logic * fix bug * polish code * fix commit consumed offset back * refactor litePullConsumer * development save * development save * Refactor DefaultLitePullConsumer and DefaultLitePullConsumerImpl. * Polish lite pull consumer * polish lite pull consumer * polish lite pull consumer * fix seek --- .../consumer/DefaultLiteMQPullConsumer.java | 132 -- .../consumer/DefaultLitePullConsumer.java | 396 ++++++ ...ullConsumer.java => LitePullConsumer.java} | 32 +- .../store/RemoteBrokerOffsetStore.java | 1 - .../impl/consumer/AssignedMessageQueue.java | 109 +- .../consumer/DefaultLitePullConsumerImpl.java | 1069 +++++++++++++++++ .../consumer/DefaultMQPullConsumerImpl.java | 4 +- .../impl/consumer/LiteMQPullConsumerImpl.java | 469 -------- .../client/impl/consumer/ProcessQueue.java | 10 - .../impl/consumer/RebalanceLitePullImpl.java | 68 ++ .../example/simple/LitePullConsumerTest.java | 24 +- 11 files changed, 1661 insertions(+), 653 deletions(-) delete mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java rename client/src/main/java/org/apache/rocketmq/client/consumer/{LiteMQPullConsumer.java => LitePullConsumer.java} (70%) create mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java delete mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java create mode 100644 client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java deleted file mode 100644 index 6f67bcf88b..0000000000 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLiteMQPullConsumer.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.consumer; - -import java.util.Collection; -import java.util.List; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.impl.consumer.LiteMQPullConsumerImpl; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.RPCHook; - -public class DefaultLiteMQPullConsumer extends DefaultMQPullConsumer implements LiteMQPullConsumer { - private LiteMQPullConsumerImpl liteMQPullConsumer; - - /** - * Maximum amount of time in minutes a message may block the consuming thread. - */ - private long consumeTimeout = 15; - - /** - * Is auto commit offset - */ - private boolean autoCommit = true; - - private int pullThreadNumbers = 20; - - /** - * Maximum commit offset interval time in seconds. - */ - private long autoCommitInterval = 5; - - public DefaultLiteMQPullConsumer(String consumerGroup, RPCHook rpcHook) { - this.setConsumerGroup(consumerGroup); - this.liteMQPullConsumer = new LiteMQPullConsumerImpl(this, rpcHook); - } - - public DefaultLiteMQPullConsumer(String consumerGroup) { - this.setConsumerGroup(consumerGroup); - this.liteMQPullConsumer = new LiteMQPullConsumerImpl(this, null); - } - - @Override - public void start() throws MQClientException { - this.liteMQPullConsumer.start(); - } - - @Override - public void subscribe(String topic, String subExpression) throws MQClientException { - this.liteMQPullConsumer.subscribe(topic, subExpression); - } - - @Override - public void unsubscribe(String topic) { - this.liteMQPullConsumer.unsubscribe(topic); - } - - @Override - public List poll() { - return poll(this.getConsumerPullTimeoutMillis()); - } - - @Override public List poll(long timeout) { - return liteMQPullConsumer.poll(timeout); - } - - @Override - public void seek(MessageQueue messageQueue, long offset) throws MQClientException { - this.liteMQPullConsumer.seek(messageQueue, offset); - } - - @Override - public void pause(Collection messageQueues) { - this.liteMQPullConsumer.pause(messageQueues); - } - - @Override - public void resume(Collection messageQueues) { - this.liteMQPullConsumer.resume(messageQueues); - } - - @Override - public void commitSync() { - this.liteMQPullConsumer.commitSync(); - } - - public long getConsumeTimeout() { - return consumeTimeout; - } - - public void setConsumeTimeout(long consumeTimeout) { - this.consumeTimeout = consumeTimeout; - } - - public boolean isAutoCommit() { - return autoCommit; - } - - public void setAutoCommit(boolean autoCommit) { - this.autoCommit = autoCommit; - } - - public int getPullThreadNumbers() { - return pullThreadNumbers; - } - - public void setPullThreadNumbers(int pullThreadNumbers) { - this.pullThreadNumbers = pullThreadNumbers; - } - - public long getAutoCommitInterval() { - return autoCommitInterval; - } - - public void setAutoCommitInterval(long autoCommitInterval) { - this.autoCommitInterval = autoCommitInterval; - } -} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java new file mode 100644 index 0000000000..757c9665cc --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.RPCHook; + + +public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { + + private DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; + + /** + * Do the same thing for the same Group, the application must be set,and guarantee Globally unique + */ + private String consumerGroup; + + /** + * Long polling mode, the Consumer connection max suspend time, it is not recommended to modify + */ + private long brokerSuspendMaxTimeMillis = 1000 * 20; + + + /** + * Long polling mode, the Consumer connection timeout(must greater than brokerSuspendMaxTimeMillis), it is not + * recommended to modify + */ + private long consumerTimeoutMillisWhenSuspend = 1000 * 30; + + /** + * The socket timeout in milliseconds + */ + private long consumerPullTimeoutMillis = 1000 * 10; + + /** + * Consumption pattern,default is clustering + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * Message queue listener + */ + private MessageQueueListener messageQueueListener; + /** + * Offset Storage + */ + private OffsetStore offsetStore; + /** + * Topic set you want to register + */ + private Set registerTopics = new HashSet(); + /** + * Queue allocation algorithm + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); + /** + * Whether the unit of subscription group + */ + private boolean unitMode = false; + + private int maxReconsumeTimes = 16; + /** + * Maximum amount of time in minutes a message may block the consuming thread. + */ + private long consumeTimeout = 15; + + /** + * Is auto commit offset + */ + private boolean autoCommit = true; + + private int pullThreadNumbers = 20; + + /** + * Maximum commit offset interval time in seconds. + */ + private long autoCommitInterval = 5; + + /** + * Maximum number of messages pulled each time. + */ + private int pullBatchNums = 10; + + /** + * Flow control threshold for consume request, each consumer will cache at most 10000 consume requests by default. + * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit + */ + private long pullThresholdForAll = 10000; + + /** + * Consume max span offset. + */ + private int consumeMaxSpan = 2000; + + /** + * Flow control threshold on queue level, each message queue will cache at most 1000 messages by default, Consider + * the {@code pullBatchSize}, the instantaneous value may exceed the limit + */ + private int pullThresholdForQueue = 1000; + + /** + * Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default, + * Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit + * + *

+ * The size of a message only measured by message body, so it's not accurate + */ + private int pullThresholdSizeForQueue = 100; + + /** + * The socket timeout in milliseconds + */ + private long pollTimeoutMillis = 1000 * 5; + + public DefaultLitePullConsumer() { + this(null, MixAll.DEFAULT_CONSUMER_GROUP, null); + } + + public DefaultLitePullConsumer(final String consumerGroup) { + this(null, consumerGroup, null); + } + + public DefaultLitePullConsumer(RPCHook rpcHook) { + this(null, MixAll.DEFAULT_CONSUMER_GROUP, rpcHook); + } + + public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) { + this(null, consumerGroup, rpcHook); + } + + /** + * Constructor specifying namespace, consumer group and RPC hook. + * + * @param consumerGroup Consumer group. + * @param rpcHook RPC hook to execute before each remoting command. + */ + public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) { + this.namespace = namespace; + this.consumerGroup = consumerGroup; + defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this,rpcHook); + } + + @Override + public void start() throws MQClientException { + this.defaultLitePullConsumerImpl.start(); + } + + @Override + public void shutdown() { + this.defaultLitePullConsumerImpl.shutdown(); + } + + @Override + public void subscribe(String topic, String subExpression) throws MQClientException { + this.defaultLitePullConsumerImpl.subscribe(topic, subExpression); + } + + @Override + public void unsubscribe(String topic) { + this.defaultLitePullConsumerImpl.unsubscribe(topic); + } + + @Override + public void assign(Collection messageQueues) { + defaultLitePullConsumerImpl.assign(messageQueues); + } + + @Override + public List poll() { + return defaultLitePullConsumerImpl.poll(this.getPollTimeoutMillis()); + } + + @Override + public List poll(long timeout) { + return defaultLitePullConsumerImpl.poll(timeout); + } + + @Override + public void seek(MessageQueue messageQueue, long offset) throws MQClientException { + this.defaultLitePullConsumerImpl.seek(messageQueue, offset); + } + + @Override + public void pause(Collection messageQueues) { + this.defaultLitePullConsumerImpl.pause(messageQueues); + } + + @Override + public void resume(Collection messageQueues) { + this.defaultLitePullConsumerImpl.resume(messageQueues); + } + + @Override + public Collection fetchMessageQueues(String topic) throws MQClientException{ + return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic)); + } + + @Override + public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException{ + return this.defaultLitePullConsumerImpl.searchOffset(messageQueue,timestamp); + } + + @Override + public void commitSync() { + this.defaultLitePullConsumerImpl.commitSync(); + } + + @Override + public boolean isAutoCommit() { + return autoCommit; + } + + @Override + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + public long getConsumeTimeout() { + return consumeTimeout; + } + + public void setConsumeTimeout(long consumeTimeout) { + this.consumeTimeout = consumeTimeout; + } + + public int getPullThreadNumbers() { + return pullThreadNumbers; + } + + public void setPullThreadNumbers(int pullThreadNumbers) { + this.pullThreadNumbers = pullThreadNumbers; + } + + public long getAutoCommitInterval() { + return autoCommitInterval; + } + + public void setAutoCommitInterval(long autoCommitInterval) { + this.autoCommitInterval = autoCommitInterval; + } + + public int getPullBatchNums() { + return pullBatchNums; + } + + public void setPullBatchNums(int pullBatchNums) { + this.pullBatchNums = pullBatchNums; + } + + public long getPullThresholdForAll() { + return pullThresholdForAll; + } + + public void setPullThresholdForAll(long pullThresholdForAll) { + this.pullThresholdForAll = pullThresholdForAll; + } + + public int getConsumeMaxSpan() { + return consumeMaxSpan; + } + + public void setConsumeMaxSpan(int consumeMaxSpan) { + this.consumeMaxSpan = consumeMaxSpan; + } + + public int getPullThresholdForQueue() { + return pullThresholdForQueue; + } + + public void setPullThresholdForQueue(int pullThresholdForQueue) { + this.pullThresholdForQueue = pullThresholdForQueue; + } + + public int getPullThresholdSizeForQueue() { + return pullThresholdSizeForQueue; + } + + public void setPullThresholdSizeForQueue(int pullThresholdSizeForQueue) { + this.pullThresholdSizeForQueue = pullThresholdSizeForQueue; + } + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + public long getBrokerSuspendMaxTimeMillis() { + return brokerSuspendMaxTimeMillis; + } + + public long getPollTimeoutMillis() { + return pollTimeoutMillis; + } + + public void setPollTimeoutMillis(long pollTimeoutMillis) { + this.pollTimeoutMillis = pollTimeoutMillis; + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + public boolean isUnitMode() { + return unitMode; + } + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + public int getMaxReconsumeTimes() { + return maxReconsumeTimes; + } + + public void setMaxReconsumeTimes(final int maxReconsumeTimes) { + this.maxReconsumeTimes = maxReconsumeTimes; + } + + public MessageModel getMessageModel() { + return messageModel; + } + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + public String getConsumerGroup() { + return consumerGroup; + } + + public MessageQueueListener getMessageQueueListener() { + return messageQueueListener; + } + + public void setMessageQueueListener(MessageQueueListener messageQueueListener) { + this.messageQueueListener = messageQueueListener; + } + + public Set getRegisterTopics() { + return registerTopics; + } + + public void setRegisterTopics(Set registerTopics) { + this.registerTopics = withNamespace(registerTopics); + } + + public long getConsumerPullTimeoutMillis() { + return consumerPullTimeoutMillis; + } + + public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) { + this.consumerPullTimeoutMillis = consumerPullTimeoutMillis; + } + + public long getConsumerTimeoutMillisWhenSuspend() { + return consumerTimeoutMillisWhenSuspend; + } + + public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { + this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; + } + +} diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java similarity index 70% rename from client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java rename to client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java index da8d1cf1d7..ece08af61b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LiteMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -18,16 +18,28 @@ package org.apache.rocketmq.client.consumer; import java.util.Collection; import java.util.List; + import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -public interface LiteMQPullConsumer { +public interface LitePullConsumer { + + /** + * Start the consumer + */ + void start() throws MQClientException; + + /** + * Shutdown the consumer + */ + void shutdown(); + /** * Subscribe some topic * * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if - * null or * expression,meaning subscribe all + * null or * expression,meaning subscribe all */ void subscribe(final String topic, final String subExpression) throws MQClientException; @@ -38,15 +50,27 @@ public interface LiteMQPullConsumer { */ void unsubscribe(final String topic); + void assign(Collection messageQueues); + List poll(); List poll(long timeout); void seek(MessageQueue messageQueue, long offset) throws MQClientException; - void pause(Collection messageQueueCollection); + void pause(Collection messageQueues); - void resume(Collection partitions); + boolean isAutoCommit(); + + void setAutoCommit(boolean autoCommit); + + void resume(Collection messageQueues); + + Collection fetchMessageQueues(String topic) throws MQClientException; + + Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException; void commitSync(); + + } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java index c1524e1077..63dc525e7e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -204,7 +204,6 @@ public class RemoteBrokerOffsetStore implements OffsetStore { MQBrokerException, InterruptedException, MQClientException { FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); if (null == findBrokerResult) { - this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index fb0ca79b4e..a3c5da1e79 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -19,25 +19,33 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Collection; import java.util.Iterator; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.common.message.MessageQueue; public class AssignedMessageQueue { private ConcurrentHashMap assignedMessageQueueState; + private RebalanceImpl rebalanceImpl; + public AssignedMessageQueue() { assignedMessageQueueState = new ConcurrentHashMap(); } + public void setRebalanceImpl(RebalanceImpl rebalanceImpl) { + this.rebalanceImpl = rebalanceImpl; + } + + public Collection messageQueues(){ + return assignedMessageQueueState.keySet(); + } + public boolean isPaused(MessageQueue messageQueue) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); if (messageQueueStat != null) { return messageQueueStat.isPaused(); } - return false; + return true; } public void pause(Collection messageQueues) { @@ -58,24 +66,60 @@ public class AssignedMessageQueue { } } - public long getNextOffset(MessageQueue messageQueue) throws MQClientException { + public ProcessQueue getProcessQueue(MessageQueue messageQueue) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (assignedMessageQueueState.get(messageQueue) != null) { + if (messageQueueStat != null) { + return messageQueueStat.getProcessQueue(); + } + return null; + } + + public long getNextOffset(MessageQueue messageQueue) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { return messageQueueStat.getNextOffset(); } return -1; } - public void updateNextOffset(MessageQueue messageQueue, long offset) throws MQClientException { + public void updateNextOffset(MessageQueue messageQueue, long offset) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueue == null) { - messageQueueStat = new MessageQueueStat(messageQueue, offset); - assignedMessageQueueState.putIfAbsent(messageQueue, messageQueueStat); + if (messageQueueStat != null) { + messageQueueStat.setNextOffset(offset); } - assignedMessageQueueState.get(messageQueue).setNextOffset(offset); } - public void updateAssignedMessageQueue(Set assigned) { + public long getConusmerOffset(MessageQueue messageQueue) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + return messageQueueStat.getConsumeOffset(); + } + return -1; + } + + public void updateConsumeOffset(MessageQueue messageQueue, long offset) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + messageQueueStat.setConsumeOffset(offset); + } + } + + public void setSeekOffset(MessageQueue messageQueue, long offset) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + messageQueueStat.setSeekOffset(offset); + } + } + + public long getSeekOffset(MessageQueue messageQueue) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + return messageQueueStat.getSeekOffset(); + } + return -1; + } + + public void updateAssignedMessageQueue(Collection assigned) { synchronized (this.assignedMessageQueueState) { Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); while (it.hasNext()) { @@ -87,7 +131,13 @@ public class AssignedMessageQueue { for (MessageQueue messageQueue : assigned) { if (!this.assignedMessageQueueState.containsKey(messageQueue)) { - MessageQueueStat messageQueueStat = new MessageQueueStat(messageQueue); + MessageQueueStat messageQueueStat; + if (rebalanceImpl != null && rebalanceImpl.processQueueTable.get(messageQueue) != null) { + messageQueueStat = new MessageQueueStat(messageQueue, rebalanceImpl.processQueueTable.get(messageQueue)); + } else { + ProcessQueue processQueue = new ProcessQueue(); + messageQueueStat = new MessageQueueStat(messageQueue, processQueue); + } this.assignedMessageQueueState.put(messageQueue, messageQueueStat); } } @@ -108,16 +158,15 @@ public class AssignedMessageQueue { public class MessageQueueStat { private MessageQueue messageQueue; + private ProcessQueue processQueue; private boolean paused = false; private long nextOffset = -1; + private long consumeOffset = -1; + private volatile long seekOffset = -1; - public MessageQueueStat(MessageQueue messageQueue) { + public MessageQueueStat(MessageQueue messageQueue, ProcessQueue processQueue) { this.messageQueue = messageQueue; - } - - public MessageQueueStat(MessageQueue messageQueue, long nextOffset) { - this.messageQueue = messageQueue; - this.nextOffset = nextOffset; + this.processQueue = processQueue; } public MessageQueue getMessageQueue() { @@ -143,5 +192,29 @@ public class AssignedMessageQueue { public void setNextOffset(long nextOffset) { this.nextOffset = nextOffset; } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + public void setProcessQueue(ProcessQueue processQueue) { + this.processQueue = processQueue; + } + + public long getConsumeOffset() { + return consumeOffset; + } + + public void setConsumeOffset(long consumeOffset) { + this.consumeOffset = consumeOffset; + } + + public long getSeekOffset() { + return seekOffset; + } + + public void setSeekOffset(long seekOffset) { + this.seekOffset = seekOffset; + } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java new file mode 100644 index 0000000000..95e218f0dd --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -0,0 +1,1069 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.impl.consumer; + +import java.util.List; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.Collection; +import java.util.Collections; +import java.util.TreeMap; +import java.util.HashSet; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; + +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.rocketmq.client.Validators; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.MessageSelector; +import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.hook.ConsumeMessageContext; +import org.apache.rocketmq.client.hook.ConsumeMessageHook; +import org.apache.rocketmq.client.hook.FilterMessageHook; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.ServiceState; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.help.FAQUrl; + +import org.apache.rocketmq.common.message.*; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.sysflag.PullSysFlag; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingException; + +public class DefaultLitePullConsumerImpl implements MQConsumerInner { + + private final InternalLogger log = ClientLogger.getLog(); + + private final long consumerStartTimestamp = System.currentTimeMillis(); + + private final RPCHook rpcHook; + + private final ArrayList consumeMessageHookList = new ArrayList(); + + private final ArrayList filterMessageHookList = new ArrayList(); + + private volatile ServiceState serviceState = ServiceState.CREATE_JUST; + + protected MQClientInstance mQClientFactory; + + private PullAPIWrapper pullAPIWrapper; + + private OffsetStore offsetStore; + + private RebalanceImpl rebalanceImpl = new RebalanceLitePullImpl(this); + + private enum SubscriptionType { + NONE, SUBSCRIBE, ASSIGN + } + + private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running."; + + private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Cannot select two subscription types at the same time."; + /** + * the type of subscription + */ + private SubscriptionType subscriptionType = SubscriptionType.NONE; + /** + * Delay some time when exception occur + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 1000; + /** + * Flow control interval + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50; + /** + * Delay some time when suspend pull service + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + + private DefaultLitePullConsumer defaultLitePullConsumer; + + private final ConcurrentMap taskTable = + new ConcurrentHashMap(); + + private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue(); + + private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); + + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + + private long consumeRequestFlowControlTimes = 0L; + + private long queueFlowControlTimes = 0L; + + private long queueMaxSpanFlowControlTimes = 0L; + + private long nextAutoCommitDeadline = -1L; + + public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { + + this.defaultLitePullConsumer = defaultLitePullConsumer; + this.rpcHook = rpcHook; + + } + + private void checkServiceState() { + if (!(this.serviceState == ServiceState.RUNNING)) + throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); + } + + private synchronized void setSubscriptionType(SubscriptionType type) { + if (this.subscriptionType == SubscriptionType.NONE) + this.subscriptionType = type; + else if (this.subscriptionType != type) + throw new IllegalStateException(SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE); + } + + private void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { + this.assignedMessageQueue.updateAssignedMessageQueue(assignedMessageQueue); + updatePullTask(topic, assignedMessageQueue); + } + + private void updatePullTask(String topic, Set mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + startPullTask(mqNewSet); + } + + class MessageQueueListenerImpl implements MessageQueueListener { + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageModel messageModel = defaultLitePullConsumer.getMessageModel(); + switch (messageModel) { + case BROADCASTING: + updateAssignedMessageQueue(topic, mqAll); + break; + case CLUSTERING: + updateAssignedMessageQueue(topic, mqDivided); + break; + default: + break; + } + } + } + + private int nextPullBatchNums() { + return Math.min(this.defaultLitePullConsumer.getPullBatchNums(), consumeRequestCache.remainingCapacity()); + } + + public synchronized void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultLitePullConsumer.getConsumerGroup()); + this.mQClientFactory.shutdown(); + log.info("the consumer [{}] shutdown OK", this.defaultLitePullConsumer.getConsumerGroup()); + scheduledThreadPoolExecutor.shutdown(); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + public synchronized void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + this.copySubscription(); + + if (this.defaultLitePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultLitePullConsumer.changeInstanceNameToPID(); + } + + this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); + + this.rebalanceImpl.setConsumerGroup(this.defaultLitePullConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultLitePullConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + + this.pullAPIWrapper = new PullAPIWrapper( + mQClientFactory, + this.defaultLitePullConsumer.getConsumerGroup(), isUnitMode()); + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + + if (this.defaultLitePullConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultLitePullConsumer.getOffsetStore(); + } else { + switch (this.defaultLitePullConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + default: + break; + } + this.defaultLitePullConsumer.setOffsetStore(this.offsetStore); + } + + this.offsetStore.load(); + + boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + + throw new MQClientException("The consumer group[" + this.defaultLitePullConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), + null); + } + + mQClientFactory.start(); + + final String group = this.defaultLitePullConsumer.getConsumerGroup(); + + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.defaultLitePullConsumer.getPullThreadNumbers(), + new ThreadFactoryImpl("PullMsgThread-" + group) + ); + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + if (subscriptionType == SubscriptionType.ASSIGN) { + updateAssignPullTask(assignedMessageQueue.messageQueues()); + } + + log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PullConsumer service state not OK, maybe started once, " + + this.serviceState + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), + null); + default: + break; + } + } + + private void checkConfig() throws MQClientException { + // check consumerGroup + Validators.checkGroup(this.defaultLitePullConsumer.getConsumerGroup()); + + // consumerGroup + if (null == this.defaultLitePullConsumer.getConsumerGroup()) { + throw new MQClientException( + "consumerGroup is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // consumerGroup + if (this.defaultLitePullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException( + "consumerGroup can not equal " + + MixAll.DEFAULT_CONSUMER_GROUP + + ", please specify another one." + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // messageModel + if (null == this.defaultLitePullConsumer.getMessageModel()) { + throw new MQClientException( + "messageModel is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // allocateMessageQueueStrategy + if (null == this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException( + "allocateMessageQueueStrategy is null" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + + // allocateMessageQueueStrategy + if (this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis()) { + throw new MQClientException( + "Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis" + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + null); + } + } + + private void copySubscription() throws MQClientException { + try { + Set registerTopics = this.defaultLitePullConsumer.getRegisterTopics(); + if (registerTopics != null) { + for (final String topic : registerTopics) { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + } + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + private void startPullTask(Collection mqSet) { + for (MessageQueue messageQueue : mqSet) { + if (!this.taskTable.containsKey(messageQueue)) { + PullTaskImpl pullTask = new PullTaskImpl(messageQueue); + this.taskTable.put(messageQueue, pullTask); + this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS); + } + } + } + + private void updateAssignPullTask(Collection mqNewSet) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + + startPullTask(mqNewSet); + } + + private void updateTopicSubscribeInfoWhenSubscriptionChanged() { + Map subTable = rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + } + } + + public synchronized void subscribe(String topic, String subExpression) throws MQClientException { + try { + setSubscriptionType(SubscriptionType.SUBSCRIBE); + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), + topic, subExpression); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl); + if (serviceState == ServiceState.RUNNING) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + } catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + public synchronized void unsubscribe(final String topic) { + this.rebalanceImpl.getSubscriptionInner().remove(topic); + //can be delete + removePullTaskCallback(topic); + assignedMessageQueue.removeAssignedMessageQueue(topic); + } + + public synchronized void assign(Collection messageQueues) { + setSubscriptionType(SubscriptionType.ASSIGN); + assignedMessageQueue.updateAssignedMessageQueue(messageQueues); + if (serviceState == ServiceState.RUNNING) { + updateAssignPullTask(messageQueues); + } + } + + private void maybeAutoCommit() { + long now = System.currentTimeMillis(); + if (now >= nextAutoCommitDeadline) { + commitAll(); + nextAutoCommitDeadline = now + defaultLitePullConsumer.getAutoCommitInterval() * 1000; + } + } + + public List poll(long timeout) { + try { + checkServiceState(); + if (defaultLitePullConsumer.isAutoCommit()) { + maybeAutoCommit(); + } + long endTime = System.currentTimeMillis() + timeout; + ConsumeRequest consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); + while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) { + consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); + if ((endTime - System.currentTimeMillis()) <= 0) + break; + } + if (consumeRequest != null && !consumeRequest.getProcessQueue().isDropped()) { + List messages = consumeRequest.getMessageExts(); + long offset = consumeRequest.getProcessQueue().removeMessage(messages); + assignedMessageQueue.updateConsumeOffset(consumeRequest.getMessageQueue(), offset); + return messages; + } + } catch (InterruptedException ignore) { + + } + return null; + } + + public void pause(Collection messageQueues) { + assignedMessageQueue.pause(messageQueues); + } + + public void resume(Collection messageQueues) { + assignedMessageQueue.resume(messageQueues); + } + + public synchronized void seek(MessageQueue messageQueue, long offset) throws MQClientException { + if (offset < minOffset(messageQueue) || offset > maxOffset(messageQueue)) + throw new MQClientException("Seek offset illegal", null); + try { + assignedMessageQueue.setSeekOffset(messageQueue, offset); + updateConsumeOffset(messageQueue, offset); + updateConsumeOffsetToBroker(messageQueue, offset, false); + } catch (Exception e) { + log.error("Seek offset failed.", e); + } + } + + public long maxOffset(MessageQueue mq) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + public long minOffset(MessageQueue mq) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + public void removePullTaskCallback(final String topic) { + removePullTask(topic); + } + + public void removePullTask(final String topic) { + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + + public synchronized void commitSync() { + try { + for (MessageQueue messageQueue : assignedMessageQueue.messageQueues()) { + long consumerOffset = assignedMessageQueue.getConusmerOffset(messageQueue); + if (consumerOffset != -1) { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + long preConsumerOffset = this.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY); + if (processQueue != null && !processQueue.isDropped() && consumerOffset != preConsumerOffset) { + updateConsumeOffset(messageQueue, consumerOffset); + updateConsumeOffsetToBroker(messageQueue, consumerOffset, false); + } + } + } + } catch (Exception e) { + log.error("An error occurred when update consume offset synchronously.", e); + } + } + + public synchronized void commitAll() { + try { + for (MessageQueue messageQueue : assignedMessageQueue.messageQueues()) { + long consumerOffset = assignedMessageQueue.getConusmerOffset(messageQueue); + if (consumerOffset != -1) { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + long preConsumerOffset = this.getOffsetStore().readOffset(messageQueue, ReadOffsetType.READ_FROM_MEMORY); + if (processQueue != null && !processQueue.isDropped() && consumerOffset != preConsumerOffset) { + updateConsumeOffset(messageQueue, consumerOffset); + updateConsumeOffsetToBroker(messageQueue, consumerOffset, true); + } + } + } + } catch (Exception e) { + log.error("An error occurred when update consume offset Automatically."); + } + } + + private void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { + if (assignedMessageQueue.getSeekOffset(remoteQueue) == -1) { + assignedMessageQueue.updateNextOffset(remoteQueue, nextPullOffset); + } + } + + private void submitConsumeRequest(ConsumeRequest consumeRequest) { + try { + consumeRequestCache.put(consumeRequest); + } catch (InterruptedException ex) { + log.error("Submit consumeRequest error", ex); + } + } + + private long fetchConsumeOffset(MessageQueue mq, boolean fromStore) { + checkServiceState(); + return this.offsetStore.readOffset(mq, fromStore ? ReadOffsetType.READ_FROM_STORE : ReadOffsetType.MEMORY_FIRST_THEN_STORE); + } + + private long nextPullOffset(MessageQueue remoteQueue) { + long offset = -1; + long seekOffset = assignedMessageQueue.getSeekOffset(remoteQueue); + if (seekOffset != -1) { + offset = seekOffset; + assignedMessageQueue.setSeekOffset(remoteQueue, -1); + assignedMessageQueue.updateNextOffset(remoteQueue,offset); + } else { + offset = assignedMessageQueue.getNextOffset(remoteQueue); + if (offset == -1) { + offset = fetchConsumeOffset(remoteQueue, false); + assignedMessageQueue.updateNextOffset(remoteQueue, offset); + assignedMessageQueue.updateConsumeOffset(remoteQueue, offset); + } + } + + return offset; + } + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + checkServiceState(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + public class PullTaskImpl implements Runnable { + private final MessageQueue messageQueue; + private volatile boolean cancelled = false; + + public PullTaskImpl(final MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + @Override + public void run() { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + + if (processQueue == null && processQueue.isDropped()) { + log.info("the message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); + return; + } + + if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchNums() > defaultLitePullConsumer.getPullThresholdForAll()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((consumeRequestFlowControlTimes++ % 1000) == 0) + log.warn("the consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); + return; + } + + long cachedMessageCount = processQueue.getMsgCount().get(); + long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); + + if (cachedMessageCount > defaultLitePullConsumer.getPullThresholdForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (cachedMessageSizeInMiB > defaultLitePullConsumer.getPullThresholdSizeForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (processQueue.getMaxSpan() > defaultLitePullConsumer.getConsumeMaxSpan()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { + log.warn( + "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", + processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes); + } + return; + } + + if (!this.isCancelled()) { + if (assignedMessageQueue.isPaused(messageQueue)) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS); + log.debug("Message Queue: {} has been paused!", messageQueue); + return; + } + String subExpression = null; + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + String topic = this.messageQueue.getTopic(); + subExpression = rebalanceImpl.getSubscriptionInner().get(topic).getSubString(); + } + long offset = nextPullOffset(messageQueue); + long pullDelayTimeMills = 0; + try { + PullResult pullResult = pull(messageQueue, subExpression, offset, nextPullBatchNums()); + switch (pullResult.getPullStatus()) { + case FOUND: + processQueue.putMessage(pullResult.getMsgFoundList()); + submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + pullDelayTimeMills = 0; + break; + case NO_NEW_MSG: + pullDelayTimeMills = 100; + case OFFSET_ILLEGAL: + //TODO + log.warn("the pull request offset illegal, {}", pullResult.toString()); + break; + default: + break; + } + updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); + } catch (Throwable e) { + pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION; + e.printStackTrace(); + log.error("An error occurred in pull message process.", e); + } + + if (!this.isCancelled()) { + scheduledThreadPoolExecutor.schedule(this, pullDelayTimeMills, TimeUnit.MILLISECONDS); + } else { + log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); + } + } + } + + public boolean isCancelled() { + return cancelled; + } + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + private PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return pull(mq, subExpression, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); + } + + private PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); + } + + private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return pull(mq, messageSelector, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); + } + + private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); + } + + private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression) + throws MQClientException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + try { + return FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), + mq.getTopic(), subExpression); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + } + + private SubscriptionData getSubscriptionData(MessageQueue mq, MessageSelector messageSelector) + throws MQClientException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + try { + return FilterAPI.build(mq.getTopic(), + messageSelector.getExpression(), messageSelector.getExpressionType()); + } catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + } + + private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, + boolean block, + long timeout) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + this.subscriptionAutomatically(mq.getTopic()); + + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); + + long timeoutMillis = block ? this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; + + boolean isTagType = ExpressionType.isTagType(subscriptionData.getExpressionType()); + PullResult pullResult = this.pullAPIWrapper.pullKernelImpl( + mq, + subscriptionData.getSubString(), + subscriptionData.getExpressionType(), + isTagType ? 0L : subscriptionData.getSubVersion(), + offset, + maxNums, + sysFlag, + 0, + this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis(), + timeoutMillis, + CommunicationMode.SYNC, + null + ); + this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + //If namespace not null , reset Topic without namespace. + this.resetTopic(pullResult.getMsgFoundList()); + if (!this.consumeMessageHookList.isEmpty()) { + ConsumeMessageContext consumeMessageContext = null; + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); + consumeMessageContext.setConsumerGroup(this.groupName()); + consumeMessageContext.setMq(mq); + consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); + consumeMessageContext.setSuccess(false); + this.executeHookBefore(consumeMessageContext); + consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + consumeMessageContext.setSuccess(true); + this.executeHookAfter(consumeMessageContext); + } + return pullResult; + } + + private void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } catch (Throwable ignored) { + } + } + } + } + + private void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } catch (Throwable ignored) { + } + } + } + } + + public void resetTopic(List msgList) { + if (null == msgList || msgList.size() == 0) { + return; + } + + //If namespace not null , reset Topic without namespace. + for (MessageExt messageExt : msgList) { + if (null != this.defaultLitePullConsumer.getNamespace()) { + messageExt.setTopic(NamespaceUtil.withoutNamespace(messageExt.getTopic(), this.defaultLitePullConsumer.getNamespace())); + } + } + + } + + public void subscriptionAutomatically(final String topic) { + if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) { + try { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData); + } catch (Exception ignore) { + } + } + } + + public void updateConsumeOffset(MessageQueue mq, long offset) { + checkServiceState(); + this.offsetStore.updateOffset(mq, offset, false); + } + + @Override + public String groupName() { + return this.defaultLitePullConsumer.getConsumerGroup(); + } + + @Override + public MessageModel messageModel() { + return this.defaultLitePullConsumer.getMessageModel(); + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public ConsumeFromWhere consumeFromWhere() { + return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + } + + @Override + public Set subscriptions() { + Set result = new HashSet(); + + Set topics = this.defaultLitePullConsumer.getRegisterTopics(); + if (topics != null) { + synchronized (topics) { + for (String t : topics) { + SubscriptionData ms = null; + try { + ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL); + } catch (Exception e) { + log.error("parse subscription error", e); + } + ms.setSubVersion(0L); + result.add(ms); + } + } + } + + return result; + } + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null) { + this.rebalanceImpl.doRebalance(false); + } + } + + @Override + public void persistConsumerOffset() { + try { + checkServiceState(); + Set mqs = new HashSet(); + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + mqs.addAll(allocateMq); + this.offsetStore.persistAll(mqs); + } catch (Exception e) { + log.error("group: " + this.defaultLitePullConsumer.getConsumerGroup() + " persistConsumerOffset exception", e); + } + } + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info); + } + } + } + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + @Override + public boolean isUnitMode() { + return this.defaultLitePullConsumer.isUnitMode(); + } + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + Properties prop = MixAll.object2Properties(this.defaultLitePullConsumer); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); + info.setProperties(prop); + + info.getSubscriptionSet().addAll(this.subscriptions()); + return info; + } + + private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + sendMessageBack(msg, delayLevel, brokerName, this.defaultLitePullConsumer.getConsumerGroup()); + } + + private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) + : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + + if (UtilAll.isBlank(consumerGroup)) { + consumerGroup = this.defaultLitePullConsumer.getConsumerGroup(); + } + + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLevel, 3000, + this.defaultLitePullConsumer.getMaxReconsumeTimes()); + } catch (Exception e) { + log.error("sendMessageBack Exception, " + this.defaultLitePullConsumer.getConsumerGroup(), e); + + Message newMsg = new Message(MixAll.getRetryTopic(this.defaultLitePullConsumer.getConsumerGroup()), msg.getBody()); + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); + MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(this.defaultLitePullConsumer.getMaxReconsumeTimes())); + newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } finally { + msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultLitePullConsumer.getNamespace())); + } + } + + private void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway); + } + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + public DefaultLitePullConsumer getDefaultLitePullConsumer() { + return defaultLitePullConsumer; + } + + public Set fetchMessageQueues(String topic) throws MQClientException { + checkServiceState(); + // check if has info in memory, otherwise invoke api. + Set result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + if (null == result) { + result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + } + + return parseMessageQueues(result); + } + + private Set parseMessageQueues(Set queueSet) { + Set resultQueues = new HashSet(); + for (MessageQueue messageQueue : queueSet) { + String userTopic = NamespaceUtil.withoutNamespace(messageQueue.getTopic(), + this.defaultLitePullConsumer.getNamespace()); + resultQueues.add(new MessageQueue(userTopic, messageQueue.getBrokerName(), messageQueue.getQueueId())); + } + return resultQueues; + } + + public class ConsumeRequest { + private final List messageExts; + private final MessageQueue messageQueue; + private final ProcessQueue processQueue; + private long startConsumeTimeMillis; + + public ConsumeRequest(final List messageExts, final MessageQueue messageQueue, + final ProcessQueue processQueue) { + this.messageExts = messageExts; + this.messageQueue = messageQueue; + this.processQueue = processQueue; + } + + public List getMessageExts() { + return messageExts; + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + public long getStartConsumeTimeMillis() { + return startConsumeTimeMillis; + } + + public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) { + this.startConsumeTimeMillis = startConsumeTimeMillis; + } + } +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index bc0884a4ef..3c98385e13 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -68,7 +68,7 @@ import org.apache.rocketmq.remoting.exception.RemotingException; public class DefaultMQPullConsumerImpl implements MQConsumerInner { private final InternalLogger log = ClientLogger.getLog(); - protected final DefaultMQPullConsumer defaultMQPullConsumer; + private final DefaultMQPullConsumer defaultMQPullConsumer; private final long consumerStartTimestamp = System.currentTimeMillis(); private final RPCHook rpcHook; private final ArrayList consumeMessageHookList = new ArrayList(); @@ -77,7 +77,7 @@ public class DefaultMQPullConsumerImpl implements MQConsumerInner { protected MQClientInstance mQClientFactory; private PullAPIWrapper pullAPIWrapper; private OffsetStore offsetStore; - protected RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); + private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); public DefaultMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { this.defaultMQPullConsumer = defaultMQPullConsumer; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java deleted file mode 100644 index ab229e444e..0000000000 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/LiteMQPullConsumerImpl.java +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.client.impl.consumer; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; - -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.rocketmq.client.consumer.DefaultLiteMQPullConsumer; -import org.apache.rocketmq.client.consumer.MessageQueueListener; -import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.common.filter.FilterAPI; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; -import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.logging.InternalLogger; -import org.apache.rocketmq.remoting.RPCHook; - -public class LiteMQPullConsumerImpl extends DefaultMQPullConsumerImpl { - - private final InternalLogger log = ClientLogger.getLog(); - - private DefaultLiteMQPullConsumer defaultLiteMQPullConsumer; - - private final ConcurrentMap taskTable = - new ConcurrentHashMap(); - - private AssignedMessageQueue assignedMessageQueue = new AssignedMessageQueue(); - - private volatile Set consumedSet = new HashSet(); - - private final BlockingQueue consumeRequestCache = new LinkedBlockingQueue(); - - private final ScheduledExecutorService cleanExpireMsgExecutors; - - private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; - - private ScheduledExecutorService autoCommitExecutors; - - private final ThreadLocal preConsumeRequestLocal = new ThreadLocal(); - - public LiteMQPullConsumerImpl(final DefaultLiteMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { - super(defaultMQPullConsumer, rpcHook); - this.defaultLiteMQPullConsumer = defaultMQPullConsumer; - this.cleanExpireMsgExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( - "Lite_CleanExpireMsgScheduledThread_")); - this.autoCommitExecutors = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( - "Lite_AutoCommitScheduledThread_")); - - } - - public void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { - this.assignedMessageQueue.updateAssignedMessageQueue(assignedMessageQueue); - updatePullTask(topic, assignedMessageQueue); - } - - public void updatePullTask(String topic, Set mqNewSet) { - Iterator> it = this.taskTable.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry next = it.next(); - if (next.getKey().getTopic().equals(topic)) { - if (!mqNewSet.contains(next.getKey())) { - next.getValue().setCancelled(true); - it.remove(); - } - } - } - - for (MessageQueue messageQueue : mqNewSet) { - if (!this.taskTable.containsKey(messageQueue)) { - PullTaskImpl pullTask = new PullTaskImpl(messageQueue); - this.taskTable.put(messageQueue, pullTask); - this.scheduledThreadPoolExecutor.schedule(pullTask, 0, TimeUnit.MILLISECONDS); - } - } - } - - class MessageQueueListenerImpl implements MessageQueueListener { - @Override - public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { - MessageModel messageModel = defaultMQPullConsumer.getMessageModel(); - switch (messageModel) { - case BROADCASTING: - updateAssignedMessageQueue(topic, mqAll); - break; - case CLUSTERING: - updateAssignedMessageQueue(topic, mqDivided); - break; - default: - break; - } - } - } - - int nextPullBatchNums() { - return Math.min(10, consumeRequestCache.remainingCapacity()); - } - - @Override - public synchronized void start() throws MQClientException { - this.defaultMQPullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); - super.start(); - final String group = this.defaultMQPullConsumer.getConsumerGroup(); - this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - this.defaultLiteMQPullConsumer.getPullThreadNumbers(), - new ThreadFactoryImpl("PullMsgThread-" + group) - ); - this.cleanExpireMsgExecutors.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - cleanExpireMsg(); - } - }, this.defaultLiteMQPullConsumer.getConsumeTimeout(), this.defaultLiteMQPullConsumer.getConsumeTimeout(), TimeUnit.MINUTES); - this.autoCommitExecutors.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - if (defaultLiteMQPullConsumer.isAutoCommit()) { - commitAll(); - } - } - }, this.defaultLiteMQPullConsumer.getAutoCommitInterval(), this.defaultLiteMQPullConsumer.getAutoCommitInterval(), TimeUnit.SECONDS); - updateTopicSubscribeInfoWhenSubscriptionChanged(); - } - - private void updateTopicSubscribeInfoWhenSubscriptionChanged() { - Map subTable = rebalanceImpl.getSubscriptionInner(); - if (subTable != null) { - for (final Map.Entry entry : subTable.entrySet()) { - final String topic = entry.getKey(); - this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); - } - } - } - - public List poll(long timeout) { - try { - addToConsumed(preConsumeRequestLocal.get()); - ConsumeRequest consumeRequest = consumeRequestCache.poll(timeout, TimeUnit.MILLISECONDS); - preConsumeRequestLocal.set(consumeRequest); - if (consumeRequest != null) { - List messages = consumeRequest.getMessageExts(); - for (MessageExt messageExt : messages) { - MessageAccessor.setConsumeStartTimeStamp(messageExt, String.valueOf(consumeRequest.getStartConsumeTimeMillis())); - } - consumeRequest.setStartConsumeTimeMillis(System.currentTimeMillis()); - return messages; - } - } catch (InterruptedException e) { - log.error("poll ComsumeRequest error.", e); - } - return null; - } - - public void pause(Collection messageQueues) { - assignedMessageQueue.pause(messageQueues); - } - - public void resume(Collection messageQueues) { - assignedMessageQueue.resume(messageQueues); - } - - public void seek(MessageQueue messageQueue, long offset) throws MQClientException { - this.updatePullOffset(messageQueue, offset); - try { - updateConsumeOffset(messageQueue, offset); - } catch (MQClientException ex) { - log.error("Seek offset to remote message queue error!", ex); - throw ex; - } - } - - public void unsubscribe(final String topic) { - super.unsubscribe(topic); - removePullTaskCallback(topic); - assignedMessageQueue.removeAssignedMessageQueue(topic); - } - - public void removePullTaskCallback(final String topic) { - removePullTask(topic); - } - - public void removePullTask(final String topic) { - synchronized (this.taskTable) { - Iterator> it = this.taskTable.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry next = it.next(); - if (next.getKey().getTopic().equals(topic)) { - next.getValue().setCancelled(true); - it.remove(); - } - } - } - } - - public void commitSync() { - addToConsumed(preConsumeRequestLocal.get()); - preConsumeRequestLocal.set(null); - commitAll(); - } - - public void commitAll() { - Set consumedRequests; - synchronized (this.consumedSet) { - consumedRequests = this.consumedSet; - this.consumedSet = new HashSet(); - } - for (ConsumeRequest consumeRequest : consumedRequests) { - consumeRequest.getProcessQueue().removeMessage(consumeRequest.messageExts); - } - Set> entrySet = this.rebalanceImpl.getProcessQueueTable().entrySet(); - for (Map.Entry entry : entrySet) { - try { - long consumeOffset = entry.getValue().getConsumeOffset(); - if (consumeOffset != -1) - updateConsumeOffset(entry.getKey(), consumeOffset); - } catch (MQClientException e) { - log.error("A error occurred in update consume offset process.", e); - } - } - this.getOffsetStore().persistAll(this.rebalanceImpl.getProcessQueueTable().keySet()); - } - - private void commit(final MessageQueue messageQueue, final ProcessQueue processQueue, final MessageExt messageExt) { - long offset = processQueue.removeMessage(Collections.singletonList(messageExt)); - try { - updateConsumeOffset(messageQueue, offset); - } catch (MQClientException e) { - log.error("An error occurred in update consume offset process.", e); - } - } - - public void subscribe(String topic, String subExpression) throws MQClientException { - try { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultMQPullConsumer.getConsumerGroup(), - topic, subExpression); - this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); - if (this.mQClientFactory != null) { - this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); - } - } catch (Exception e) { - throw new MQClientException("subscription exception", e); - } - } - - private void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { - try { - assignedMessageQueue.updateNextOffset(remoteQueue, nextPullOffset); - } catch (MQClientException e) { - log.error("A error occurred in update consume: {} offset process.", remoteQueue, e); - } - } - - private void addToConsumed(ConsumeRequest consumeRequest) { - if (consumeRequest != null) { - synchronized (this.consumedSet) { - if (!consumedSet.contains(consumeRequest)) - consumedSet.add(consumeRequest); - } - } - } - - private void submitConsumeRequest(ConsumeRequest consumeRequest) { - try { - consumeRequestCache.put(consumeRequest); - } catch (InterruptedException ex) { - log.error("Submit consumeRequest error", ex); - } - } - - private long nextPullOffset(MessageQueue remoteQueue) { - long offset = -1; - try { - offset = assignedMessageQueue.getNextOffset(remoteQueue); - if (offset == -1) { - offset = fetchConsumeOffset(remoteQueue, false); - assignedMessageQueue.updateNextOffset(remoteQueue, offset); - } - } catch (MQClientException e) { - log.error("An error occurred in fetch consume offset process.", e); - } - return offset; - } - - private void cleanExpireMsg() { - for (final Map.Entry next : rebalanceImpl.getProcessQueueTable().entrySet()) { - ProcessQueue pq = next.getValue(); - MessageQueue mq = next.getKey(); - ReadWriteLock lockTreeMap = getLockInProcessQueue(pq); - if (lockTreeMap == null) { - log.error("Gets tree map lock in process queue error of message queue:", mq); - return; - } - - TreeMap msgTreeMap = pq.getMsgTreeMap(); - - int loop = msgTreeMap.size(); - for (int i = 0; i < loop; i++) { - MessageExt msg = null; - try { - lockTreeMap.readLock().lockInterruptibly(); - try { - if (!msgTreeMap.isEmpty()) { - msg = msgTreeMap.firstEntry().getValue(); - if (System.currentTimeMillis() - Long.parseLong(MessageAccessor.getConsumeStartTimeStamp(msg)) - > this.defaultLiteMQPullConsumer.getConsumeTimeout() * 60 * 1000) { - //Expired, ack and remove it. - } else { - break; - } - } else { - break; - } - } finally { - lockTreeMap.readLock().unlock(); - } - } catch (InterruptedException e) { - log.error("Gets expired message exception", e); - } - - try { - this.defaultMQPullConsumer.sendMessageBack(msg, 3); - log.info("Send expired msg back. topic={}, msgId={}, storeHost={}, queueId={}, queueOffset={}", - msg.getTopic(), msg.getMsgId(), msg.getStoreHost(), msg.getQueueId(), msg.getQueueOffset()); - log.info("Send expired msg back."); - commit(mq, pq, msg); - } catch (Exception e) { - log.error("Send back expired msg exception", e); - } - } - } - } - - private ReadWriteLock getLockInProcessQueue(ProcessQueue pq) { - try { - return (ReadWriteLock) FieldUtils.readDeclaredField(pq, "lockTreeMap", true); - } catch (IllegalAccessException e) { - return null; - } - } - - public class PullTaskImpl implements Runnable { - private final MessageQueue messageQueue; - private volatile boolean cancelled = false; - - public PullTaskImpl(final MessageQueue messageQueue) { - this.messageQueue = messageQueue; - } - - @Override - public void run() { - String topic = this.messageQueue.getTopic(); - if (!this.isCancelled()) { - if (assignedMessageQueue.isPaused(messageQueue)) { - scheduledThreadPoolExecutor.schedule(this, 1000, TimeUnit.MILLISECONDS); - log.debug("Message Queue: {} has been paused!", messageQueue); - return; - } - SubscriptionData subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); - long offset = nextPullOffset(messageQueue); - long pullDelayTimeMills = 0; - try { - PullResult pullResult = pull(messageQueue, subscriptionData.getSubString(), offset, nextPullBatchNums()); - ProcessQueue processQueue = rebalanceImpl.getProcessQueueTable().get(messageQueue); - switch (pullResult.getPullStatus()) { - case FOUND: - if (processQueue != null) { - processQueue.putMessage(pullResult.getMsgFoundList()); - submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); - } - break; - default: - break; - } - updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); - } catch (Throwable e) { - pullDelayTimeMills = 1000; - e.printStackTrace(); - log.error("An error occurred in pull message process.", e); - } - - if (!this.isCancelled()) { - scheduledThreadPoolExecutor.schedule(this, pullDelayTimeMills, TimeUnit.MILLISECONDS); - } else { - log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); - } - } - } - - public boolean isCancelled() { - return cancelled; - } - - public void setCancelled(boolean cancelled) { - this.cancelled = cancelled; - } - - public MessageQueue getMessageQueue() { - return messageQueue; - } - } - - public class ConsumeRequest { - private final List messageExts; - private final MessageQueue messageQueue; - private final ProcessQueue processQueue; - private long startConsumeTimeMillis; - - public ConsumeRequest(final List messageExts, final MessageQueue messageQueue, - final ProcessQueue processQueue) { - this.messageExts = messageExts; - this.messageQueue = messageQueue; - this.processQueue = processQueue; - } - - public List getMessageExts() { - return messageExts; - } - - public MessageQueue getMessageQueue() { - return messageQueue; - } - - public ProcessQueue getProcessQueue() { - return processQueue; - } - - public long getStartConsumeTimeMillis() { - return startConsumeTimeMillis; - } - - public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) { - this.startConsumeTimeMillis = startConsumeTimeMillis; - } - } -} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java index e9a1c72ddf..092da9aa33 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/ProcessQueue.java @@ -433,14 +433,4 @@ public class ProcessQueue { this.lastConsumeTimestamp = lastConsumeTimestamp; } - public long getConsumeOffset() { - - if (msgTreeMap.isEmpty() && queueOffsetMax == 0L) - return -1; - - if (!msgTreeMap.isEmpty()) - return msgTreeMap.firstKey(); - else - return queueOffsetMax + 1; - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java new file mode 100644 index 0000000000..8148c7da5a --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -0,0 +1,68 @@ +package org.apache.rocketmq.client.impl.consumer; + +import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import org.apache.rocketmq.client.consumer.MessageQueueListener; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; + +import java.util.List; +import java.util.Set; + +public class RebalanceLitePullImpl extends RebalanceImpl { + + private final DefaultLitePullConsumerImpl litePullConsumerImpl; + + public RebalanceLitePullImpl(DefaultLitePullConsumerImpl litePullConsumerImpl) { + this(null, null, null, null, litePullConsumerImpl); + } + + public RebalanceLitePullImpl(String consumerGroup, MessageModel messageModel, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory, DefaultLitePullConsumerImpl litePullConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.litePullConsumerImpl = litePullConsumerImpl; + } + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageQueueListener messageQueueListener = this.litePullConsumerImpl.getDefaultLitePullConsumer().getMessageQueueListener(); + if (messageQueueListener != null) { + try { + messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided); + } catch (Throwable e) { + log.error("messageQueueChanged exception", e); + } + } + } + + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.litePullConsumerImpl.getOffsetStore().persist(mq); + this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); + return true; + } + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + @Override + public void removeDirtyOffset(final MessageQueue mq) { + this.litePullConsumerImpl.getOffsetStore().removeOffset(mq); + } + + @Override + public long computePullFromWhere(MessageQueue mq) { + return 0; + } + + @Override + public void dispatchPullRequest(List pullRequestList) { + } + + +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java index 215763bd63..488a499de1 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java @@ -16,34 +16,24 @@ */ package org.apache.rocketmq.example.simple; -import java.util.Arrays; import java.util.List; -import org.apache.rocketmq.client.consumer.DefaultLiteMQPullConsumer; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; + public class LitePullConsumerTest { public static void main(String[] args) throws Exception { - DefaultLiteMQPullConsumer litePullConsumer = new DefaultLiteMQPullConsumer("test", null); + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("test"); litePullConsumer.setNamesrvAddr("localhost:9876"); - litePullConsumer.subscribe("litepullconsumertest9", null); + litePullConsumer.setAutoCommit(true); + litePullConsumer.subscribe("test41","TagA" ); litePullConsumer.start(); - MessageQueue messageQueue = new MessageQueue("test", "IT-C02YW28FLVDL.local", 1); + int i = 0; while (true) { List messageExts = litePullConsumer.poll(); System.out.printf("%s%n", messageExts); - i++; - if (i == 3) { - System.out.printf("pause%n"); - litePullConsumer.pause(Arrays.asList(messageQueue)); - } - if (i == 10) { - System.out.printf("resume%n"); - litePullConsumer.resume(Arrays.asList(messageQueue)); - } -// - litePullConsumer.commitSync(); } + } } From 9c3b26cfd3a7b5c7b87bb13c4ab38f249107e349 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Fri, 2 Aug 2019 10:34:06 +0800 Subject: [PATCH 06/14] Polish lite pull consumer (#1359) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle * polish LiteMQPullConsumer * add flow control and polish commit logic * fix bug * polish code * fix commit consumed offset back * refactor litePullConsumer * development save * development save * Refactor DefaultLitePullConsumer and DefaultLitePullConsumerImpl. * Polish lite pull consumer * polish lite pull consumer * polish lite pull consumer * fix seek * fix seek function * polish lite pull consumer * add apache header * add test * polish test --- .../consumer/DefaultLitePullConsumer.java | 27 +- .../impl/consumer/AssignedMessageQueue.java | 41 ++- .../consumer/DefaultLitePullConsumerImpl.java | 190 +++++++------ .../impl/consumer/RebalanceLitePullImpl.java | 24 +- .../consumer/DefaultLitePullConsumerTest.java | 261 ++++++++++++++++++ .../example/simple/LitePullConsumerTest.java | 22 +- 6 files changed, 453 insertions(+), 112 deletions(-) create mode 100644 client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 757c9665cc..7f657130c4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -32,10 +32,9 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.remoting.RPCHook; - public class DefaultLitePullConsumer extends ClientConfig implements LitePullConsumer { - private DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; + private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** * Do the same thing for the same Group, the application must be set,and guarantee Globally unique @@ -47,7 +46,6 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private long brokerSuspendMaxTimeMillis = 1000 * 20; - /** * Long polling mode, the Consumer connection timeout(must greater than brokerSuspendMaxTimeMillis), it is not * recommended to modify @@ -134,10 +132,15 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon private int pullThresholdSizeForQueue = 100; /** - * The socket timeout in milliseconds + * The poll timeout in milliseconds */ private long pollTimeoutMillis = 1000 * 5; + /** + * Message pull delay in milliseconds + */ + private long pullDelayTimeMills = 0; + public DefaultLitePullConsumer() { this(null, MixAll.DEFAULT_CONSUMER_GROUP, null); } @@ -163,7 +166,7 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public DefaultLitePullConsumer(final String namespace, final String consumerGroup, RPCHook rpcHook) { this.namespace = namespace; this.consumerGroup = consumerGroup; - defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this,rpcHook); + defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook); } @Override @@ -217,13 +220,13 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon } @Override - public Collection fetchMessageQueues(String topic) throws MQClientException{ + public Collection fetchMessageQueues(String topic) throws MQClientException { return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic)); } @Override - public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException{ - return this.defaultLitePullConsumerImpl.searchOffset(messageQueue,timestamp); + public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException { + return this.defaultLitePullConsumerImpl.searchOffset(messageQueue, timestamp); } @Override @@ -393,4 +396,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; } + public long getPullDelayTimeMills() { + return pullDelayTimeMills; + } + + public void setPullDelayTimeMills(long pullDelayTimeMills) { + this.pullDelayTimeMills = pullDelayTimeMills; + } + } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index a3c5da1e79..aa8379ecdd 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.message.MessageQueue; public class AssignedMessageQueue { @@ -36,7 +37,7 @@ public class AssignedMessageQueue { this.rebalanceImpl = rebalanceImpl; } - public Collection messageQueues(){ + public Collection messageQueues() { return assignedMessageQueueState.keySet(); } @@ -52,6 +53,7 @@ public class AssignedMessageQueue { for (MessageQueue messageQueue : messageQueues) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { + messageQueueStat.getPausedLatch().reset(); messageQueueStat.setPaused(true); } } @@ -62,6 +64,7 @@ public class AssignedMessageQueue { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { messageQueueStat.setPaused(false); + messageQueueStat.getPausedLatch().reset(); } } } @@ -74,18 +77,18 @@ public class AssignedMessageQueue { return null; } - public long getNextOffset(MessageQueue messageQueue) { + public long getPullOffset(MessageQueue messageQueue) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); if (messageQueueStat != null) { - return messageQueueStat.getNextOffset(); + return messageQueueStat.getPullOffset(); } return -1; } - public void updateNextOffset(MessageQueue messageQueue, long offset) { + public void updatePullOffset(MessageQueue messageQueue, long offset) { MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); if (messageQueueStat != null) { - messageQueueStat.setNextOffset(offset); + messageQueueStat.setPullOffset(offset); } } @@ -119,12 +122,21 @@ public class AssignedMessageQueue { return -1; } + public CountDownLatch2 getPausedLatch(MessageQueue messageQueue) { + MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + if (messageQueueStat != null) { + return messageQueueStat.getPausedLatch(); + } + return null; + } + public void updateAssignedMessageQueue(Collection assigned) { synchronized (this.assignedMessageQueueState) { Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); while (it.hasNext()) { Map.Entry next = it.next(); if (!assigned.contains(next.getKey())) { + next.getValue().getProcessQueue().setDropped(true); it.remove(); } } @@ -159,10 +171,11 @@ public class AssignedMessageQueue { public class MessageQueueStat { private MessageQueue messageQueue; private ProcessQueue processQueue; - private boolean paused = false; - private long nextOffset = -1; - private long consumeOffset = -1; + private volatile boolean paused = false; + private volatile long pullOffset = -1; + private volatile long consumeOffset = -1; private volatile long seekOffset = -1; + private CountDownLatch2 pausedLatch = new CountDownLatch2(1); public MessageQueueStat(MessageQueue messageQueue, ProcessQueue processQueue) { this.messageQueue = messageQueue; @@ -185,12 +198,12 @@ public class AssignedMessageQueue { this.paused = paused; } - public long getNextOffset() { - return nextOffset; + public long getPullOffset() { + return pullOffset; } - public void setNextOffset(long nextOffset) { - this.nextOffset = nextOffset; + public void setPullOffset(long pullOffset) { + this.pullOffset = pullOffset; } public ProcessQueue getProcessQueue() { @@ -216,5 +229,9 @@ public class AssignedMessageQueue { public void setSeekOffset(long seekOffset) { this.seekOffset = seekOffset; } + + public CountDownLatch2 getPausedLatch() { + return pausedLatch; + } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 95e218f0dd..74cf644b98 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -16,27 +16,22 @@ */ package org.apache.rocketmq.client.impl.consumer; +import java.util.Collections; import java.util.List; import java.util.ArrayList; import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.Collection; -import java.util.Collections; -import java.util.TreeMap; import java.util.HashSet; import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; @@ -56,6 +51,7 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -65,7 +61,11 @@ import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; -import org.apache.rocketmq.common.message.*; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; @@ -150,7 +150,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } private void checkServiceState() { - if (!(this.serviceState == ServiceState.RUNNING)) + if (this.serviceState != ServiceState.RUNNING) throw new IllegalStateException(NOT_RUNNING_EXCEPTION_MESSAGE); } @@ -347,6 +347,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } + + } + + public PullAPIWrapper getPullAPIWrapper() { + return pullAPIWrapper; } private void copySubscription() throws MQClientException { @@ -440,16 +445,24 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public List poll(long timeout) { try { checkServiceState(); + if (timeout < 0) + throw new IllegalArgumentException("Timeout must not be negative"); + if (defaultLitePullConsumer.isAutoCommit()) { maybeAutoCommit(); } long endTime = System.currentTimeMillis() + timeout; + ConsumeRequest consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); - while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) { - consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); - if ((endTime - System.currentTimeMillis()) <= 0) - break; + + if (endTime - System.currentTimeMillis() > 0) { + while (consumeRequest != null && consumeRequest.getProcessQueue().isDropped()) { + consumeRequest = consumeRequestCache.poll(endTime - System.currentTimeMillis(), TimeUnit.MILLISECONDS); + if (endTime - System.currentTimeMillis() <= 0) + break; + } } + if (consumeRequest != null && !consumeRequest.getProcessQueue().isDropped()) { List messages = consumeRequest.getMessageExts(); long offset = consumeRequest.getProcessQueue().removeMessage(messages); @@ -471,14 +484,33 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } public synchronized void seek(MessageQueue messageQueue, long offset) throws MQClientException { - if (offset < minOffset(messageQueue) || offset > maxOffset(messageQueue)) - throw new MQClientException("Seek offset illegal", null); + if (!assignedMessageQueue.messageQueues().contains(messageQueue)) + throw new MQClientException("The message queue is not in assigned list, message queue: " + messageQueue, null); + long minOffset = minOffset(messageQueue); + long maxOffset = maxOffset(messageQueue); + if (offset < minOffset || offset > maxOffset) + throw new MQClientException("Seek offset illegal, seek offset = " + offset + ", min offset = " + minOffset + ", max offset = " + maxOffset, null); try { + assignedMessageQueue.pause(Collections.singletonList(messageQueue)); + CountDownLatch2 pausedLatch = assignedMessageQueue.getPausedLatch(messageQueue); + if (pausedLatch != null) { + pausedLatch.await(2, TimeUnit.SECONDS); + } + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + if (processQueue != null) { + processQueue.clear(); + } + Iterator iter = consumeRequestCache.iterator(); + while (iter.hasNext()) { + if (iter.next().getMessageQueue().equals(messageQueue)) + iter.remove(); + } assignedMessageQueue.setSeekOffset(messageQueue, offset); - updateConsumeOffset(messageQueue, offset); - updateConsumeOffsetToBroker(messageQueue, offset, false); + assignedMessageQueue.updateConsumeOffset(messageQueue, offset); } catch (Exception e) { log.error("Seek offset failed.", e); + } finally { + assignedMessageQueue.resume(Collections.singletonList(messageQueue)); } } @@ -545,7 +577,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { if (assignedMessageQueue.getSeekOffset(remoteQueue) == -1) { - assignedMessageQueue.updateNextOffset(remoteQueue, nextPullOffset); + assignedMessageQueue.updatePullOffset(remoteQueue, nextPullOffset); } } @@ -568,12 +600,12 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { if (seekOffset != -1) { offset = seekOffset; assignedMessageQueue.setSeekOffset(remoteQueue, -1); - assignedMessageQueue.updateNextOffset(remoteQueue,offset); + assignedMessageQueue.updatePullOffset(remoteQueue, offset); } else { - offset = assignedMessageQueue.getNextOffset(remoteQueue); + offset = assignedMessageQueue.getPullOffset(remoteQueue); if (offset == -1) { offset = fetchConsumeOffset(remoteQueue, false); - assignedMessageQueue.updateNextOffset(remoteQueue, offset); + assignedMessageQueue.updatePullOffset(remoteQueue, offset); assignedMessageQueue.updateConsumeOffset(remoteQueue, offset); } } @@ -596,78 +628,82 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { @Override public void run() { - ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); - - if (processQueue == null && processQueue.isDropped()) { - log.info("the message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); - return; - } - - if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchNums() > defaultLitePullConsumer.getPullThresholdForAll()) { - scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); - if ((consumeRequestFlowControlTimes++ % 1000) == 0) - log.warn("the consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); - return; - } - - long cachedMessageCount = processQueue.getMsgCount().get(); - long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); - - if (cachedMessageCount > defaultLitePullConsumer.getPullThresholdForQueue()) { - scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); - if ((queueFlowControlTimes++ % 1000) == 0) { - log.warn( - "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", - defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); - } - return; - } - - if (cachedMessageSizeInMiB > defaultLitePullConsumer.getPullThresholdSizeForQueue()) { - scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); - if ((queueFlowControlTimes++ % 1000) == 0) { - log.warn( - "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", - defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); - } - return; - } - - if (processQueue.getMaxSpan() > defaultLitePullConsumer.getConsumeMaxSpan()) { - scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); - if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { - log.warn( - "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", - processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes); - } - return; - } if (!this.isCancelled()) { + if (assignedMessageQueue.isPaused(messageQueue)) { + CountDownLatch2 pasuedLatch = assignedMessageQueue.getPausedLatch(messageQueue); + if (pasuedLatch != null) + pasuedLatch.countDown(); scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS); log.debug("Message Queue: {} has been paused!", messageQueue); return; } + + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + + if (processQueue == null && processQueue.isDropped()) { + log.info("the message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); + return; + } + + if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchNums() > defaultLitePullConsumer.getPullThresholdForAll()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((consumeRequestFlowControlTimes++ % 1000) == 0) + log.warn("the consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); + return; + } + + long cachedMessageCount = processQueue.getMsgCount().get(); + long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); + + if (cachedMessageCount > defaultLitePullConsumer.getPullThresholdForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (cachedMessageSizeInMiB > defaultLitePullConsumer.getPullThresholdSizeForQueue()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueFlowControlTimes++ % 1000) == 0) { + log.warn( + "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); + } + return; + } + + if (processQueue.getMaxSpan() > defaultLitePullConsumer.getConsumeMaxSpan()) { + scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); + if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { + log.warn( + "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", + processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes); + } + return; + } + String subExpression = null; if (subscriptionType == SubscriptionType.SUBSCRIBE) { String topic = this.messageQueue.getTopic(); subExpression = rebalanceImpl.getSubscriptionInner().get(topic).getSubString(); } long offset = nextPullOffset(messageQueue); - long pullDelayTimeMills = 0; + long pullDelayTimeMills = defaultLitePullConsumer.getPullDelayTimeMills(); try { PullResult pullResult = pull(messageQueue, subExpression, offset, nextPullBatchNums()); switch (pullResult.getPullStatus()) { case FOUND: - processQueue.putMessage(pullResult.getMsgFoundList()); - submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); - pullDelayTimeMills = 0; + if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty()) { + processQueue.putMessage(pullResult.getMsgFoundList()); + submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + } break; - case NO_NEW_MSG: - pullDelayTimeMills = 100; case OFFSET_ILLEGAL: - //TODO log.warn("the pull request offset illegal, {}", pullResult.toString()); break; default: @@ -1037,7 +1073,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final List messageExts; private final MessageQueue messageQueue; private final ProcessQueue processQueue; - private long startConsumeTimeMillis; public ConsumeRequest(final List messageExts, final MessageQueue messageQueue, final ProcessQueue processQueue) { @@ -1058,12 +1093,5 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return processQueue; } - public long getStartConsumeTimeMillis() { - return startConsumeTimeMillis; - } - - public void setStartConsumeTimeMillis(final long startConsumeTimeMillis) { - this.startConsumeTimeMillis = startConsumeTimeMillis; - } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java index 8148c7da5a..0b8ec67778 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceLitePullImpl.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.rocketmq.client.impl.consumer; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; @@ -10,7 +26,7 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import java.util.List; import java.util.Set; -public class RebalanceLitePullImpl extends RebalanceImpl { +public class RebalanceLitePullImpl extends RebalanceImpl { private final DefaultLitePullConsumerImpl litePullConsumerImpl; @@ -19,8 +35,8 @@ public class RebalanceLitePullImpl extends RebalanceImpl { } public RebalanceLitePullImpl(String consumerGroup, MessageModel messageModel, - AllocateMessageQueueStrategy allocateMessageQueueStrategy, - MQClientInstance mQClientFactory, DefaultLitePullConsumerImpl litePullConsumerImpl) { + AllocateMessageQueueStrategy allocateMessageQueueStrategy, + MQClientInstance mQClientFactory, DefaultLitePullConsumerImpl litePullConsumerImpl) { super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); this.litePullConsumerImpl = litePullConsumerImpl; } @@ -37,7 +53,6 @@ public class RebalanceLitePullImpl extends RebalanceImpl { } } - @Override public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { this.litePullConsumerImpl.getOffsetStore().persist(mq); @@ -64,5 +79,4 @@ public class RebalanceLitePullImpl extends RebalanceImpl { public void dispatchPullRequest(List pullRequestList) { } - } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java new file mode 100644 index 0000000000..68144c785e --- /dev/null +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.client.consumer; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.store.OffsetStore; +import org.apache.rocketmq.client.consumer.store.ReadOffsetType; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.FindBrokerResult; +import org.apache.rocketmq.client.impl.MQAdminImpl; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; +import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; +import org.apache.rocketmq.client.impl.consumer.RebalanceService; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.message.MessageClientExt; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DefaultLitePullConsumerTest { + @Spy + private MQClientInstance mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(new ClientConfig()); + + @Mock + private MQClientAPIImpl mQClientAPIImpl; + @Mock + private MQAdminImpl mQAdminImpl; + + private RebalanceImpl rebalanceImpl; + private OffsetStore offsetStore; + private DefaultLitePullConsumer litePullConsumer; + private DefaultLitePullConsumerImpl litePullConsumerImpl; + private String consumerGroup = "LitePullConsumerGroup"; + private String topic = "LitePullConsumerTest"; + private String brokerName = "BrokerA"; + + @Before + public void init() throws Exception { + String groupName = consumerGroup + System.currentTimeMillis(); + litePullConsumer = new DefaultLitePullConsumer(groupName); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + + Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); + field.setAccessible(true); + RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); + field = RebalanceService.class.getDeclaredField("waitInterval"); + field.setAccessible(true); + field.set(rebalanceService, 100); + + litePullConsumer.start(); + + field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + field.setAccessible(true); + litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(litePullConsumerImpl, mQClientFactory); + + PullAPIWrapper pullAPIWrapper = litePullConsumerImpl.getPullAPIWrapper(); + field = PullAPIWrapper.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(pullAPIWrapper, mQClientFactory); + + field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQClientAPIImpl); + + field = MQClientInstance.class.getDeclaredField("mQAdminImpl"); + field.setAccessible(true); + field.set(mQClientFactory, mQAdminImpl); + + field = DefaultLitePullConsumerImpl.class.getDeclaredField("rebalanceImpl"); + field.setAccessible(true); + rebalanceImpl = (RebalanceImpl) field.get(litePullConsumerImpl); + field = RebalanceImpl.class.getDeclaredField("mQClientFactory"); + field.setAccessible(true); + field.set(rebalanceImpl, mQClientFactory); + + offsetStore = spy(litePullConsumerImpl.getOffsetStore()); + field = DefaultLitePullConsumerImpl.class.getDeclaredField("offsetStore"); + field.setAccessible(true); + field.set(litePullConsumerImpl, offsetStore); + + when(mQClientFactory.getMQClientAPIImpl().pullMessage(anyString(), any(PullMessageRequestHeader.class), + anyLong(), any(CommunicationMode.class), nullable(PullCallback.class))) + .thenAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock mock) throws Throwable { + PullMessageRequestHeader requestHeader = mock.getArgument(1); + MessageClientExt messageClientExt = new MessageClientExt(); + messageClientExt.setTopic(topic); + messageClientExt.setQueueId(0); + messageClientExt.setMsgId("123"); + messageClientExt.setBody(new byte[] {'a'}); + messageClientExt.setOffsetMsgId("234"); + messageClientExt.setBornHost(new InetSocketAddress(8080)); + messageClientExt.setStoreHost(new InetSocketAddress(8080)); + PullResult pullResult = createPullResult(requestHeader, PullStatus.FOUND, Collections.singletonList(messageClientExt)); + return pullResult; + } + }); + + when(mQClientFactory.findBrokerAddressInSubscribe(anyString(), anyLong(), anyBoolean())).thenReturn(new FindBrokerResult("127.0.0.1:10911", false)); + + doReturn(Collections.singletonList(mQClientFactory.getClientId())).when(mQClientFactory).findConsumerIdList(anyString(), anyString()); + + doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); + } + + @After + public void terminate() { + litePullConsumer.shutdown(); + } + + @Test + public void testAssign_PollMessageSuccess() { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullDelayTimeMills(60 * 1000); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } + + @Test + public void testSubscribe_PollMessageSuccess() throws MQClientException { + litePullConsumer.setPullDelayTimeMills(60 * 1000); + litePullConsumer.subscribe(topic, "*"); + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } + + @Test + public void testSubscriptionType_AssignAndSubscribeExclusive() throws MQClientException { + try { + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.assign(Collections.singletonList(createMessageQueue())); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("Cannot select two subscription types at the same time."); + } + } + + @Test + public void testFetchMesseageQueues_FetchMessageQueuesBeforeStart() throws MQClientException { + try { + DefaultLitePullConsumer litePullConsumer = createLitePullConsumer(); + litePullConsumer.fetchMessageQueues(topic); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("The consumer not running."); + } + } + + @Test + public void testSeek_SeekOffsetIllegal() throws MQClientException { + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + try { + litePullConsumer.seek(messageQueue, -1); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("min offset = 0"); + } + + try { + litePullConsumer.seek(messageQueue, 1000); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("max offset = 100"); + } + } + + @Test + public void testSeek_MessageQueueNotInAssignList() { + try { + litePullConsumer.seek(createMessageQueue(), 0); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("The message queue is not in assigned list"); + } + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private DefaultLitePullConsumer createLitePullConsumer() { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + return litePullConsumer; + } + + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, + List messageExtList) throws Exception { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + for (MessageExt messageExt : messageExtList) { + outputStream.write(MessageDecoder.encode(messageExt, false)); + } + return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); + } +} diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java index 488a499de1..0430465cf5 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java @@ -16,23 +16,33 @@ */ package org.apache.rocketmq.example.simple; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.common.message.MessageExt; - +import org.apache.rocketmq.common.message.MessageQueue; public class LitePullConsumerTest { public static void main(String[] args) throws Exception { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("test"); - litePullConsumer.setNamesrvAddr("localhost:9876"); - litePullConsumer.setAutoCommit(true); - litePullConsumer.subscribe("test41","TagA" ); + litePullConsumer.setAutoCommit(false); litePullConsumer.start(); + Collection mqSet = litePullConsumer.fetchMessageQueues("test400"); + List list = new ArrayList<>(mqSet); + Collection assginMq = Collections.singletonList(list.get(0)); + litePullConsumer.assign(assginMq); + int size = 0; + litePullConsumer.seek(list.get(0), 26); - int i = 0; while (true) { List messageExts = litePullConsumer.poll(); - System.out.printf("%s%n", messageExts); + if (messageExts != null) { + size += messageExts.size(); + } + litePullConsumer.commitSync(); + System.out.printf("%s %d %n", messageExts, size); } } From 46288abb33e7f4cf9ca59dbfbff12be96fd8494a Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Fri, 9 Aug 2019 11:26:53 +0800 Subject: [PATCH 07/14] Polish lite pull consumer and fix bug (#1373) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle * polish LiteMQPullConsumer * add flow control and polish commit logic * fix bug * polish code * fix commit consumed offset back * refactor litePullConsumer * development save * development save * Refactor DefaultLitePullConsumer and DefaultLitePullConsumerImpl. * Polish lite pull consumer * polish lite pull consumer * polish lite pull consumer * fix seek * fix seek function * polish lite pull consumer * add apache header * add test * polish test * Make broadcast model work for litePullConsumer * Revert example/broadcast/PushConsumer.java * Add delay time when no new message * Enable long polling mode * Fix subscribe bug when rebalance * Delete useless consumeMessageHook --- .../consumer/DefaultLitePullConsumer.java | 15 +- .../impl/consumer/AssignedMessageQueue.java | 44 +++-- .../consumer/DefaultLitePullConsumerImpl.java | 157 +++++------------- .../example/broadcast/PushConsumer.java | 2 +- 4 files changed, 79 insertions(+), 139 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 7f657130c4..1858fa1780 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -17,9 +17,7 @@ package org.apache.rocketmq.client.consumer; import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.rocketmq.client.ClientConfig; import org.apache.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; @@ -69,10 +67,7 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon * Offset Storage */ private OffsetStore offsetStore; - /** - * Topic set you want to register - */ - private Set registerTopics = new HashSet(); + /** * Queue allocation algorithm */ @@ -372,14 +367,6 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon this.messageQueueListener = messageQueueListener; } - public Set getRegisterTopics() { - return registerTopics; - } - - public void setRegisterTopics(Set registerTopics) { - this.registerTopics = withNamespace(registerTopics); - } - public long getConsumerPullTimeoutMillis() { return consumerPullTimeoutMillis; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index aa8379ecdd..b21fd01e13 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Collection; import java.util.Iterator; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.message.MessageQueue; @@ -37,7 +38,7 @@ public class AssignedMessageQueue { this.rebalanceImpl = rebalanceImpl; } - public Collection messageQueues() { + public Set messageQueues() { return assignedMessageQueueState.keySet(); } @@ -130,6 +131,23 @@ public class AssignedMessageQueue { return null; } + public void updateAssignedMessageQueue(String topic, Collection assigned) { + synchronized (this.assignedMessageQueueState) { + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!assigned.contains(next.getKey())) { + System.out.printf("MessageQueue-%s is removed %n", next.getKey()); + next.getValue().getProcessQueue().setDropped(true); + it.remove(); + } + } + } + addAssignedMessageQueue(assigned); + } + } + public void updateAssignedMessageQueue(Collection assigned) { synchronized (this.assignedMessageQueueState) { Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); @@ -140,18 +158,22 @@ public class AssignedMessageQueue { it.remove(); } } + addAssignedMessageQueue(assigned); + } + } - for (MessageQueue messageQueue : assigned) { - if (!this.assignedMessageQueueState.containsKey(messageQueue)) { - MessageQueueStat messageQueueStat; - if (rebalanceImpl != null && rebalanceImpl.processQueueTable.get(messageQueue) != null) { - messageQueueStat = new MessageQueueStat(messageQueue, rebalanceImpl.processQueueTable.get(messageQueue)); - } else { - ProcessQueue processQueue = new ProcessQueue(); - messageQueueStat = new MessageQueueStat(messageQueue, processQueue); - } - this.assignedMessageQueueState.put(messageQueue, messageQueueStat); + private void addAssignedMessageQueue(Collection assigned) { + for (MessageQueue messageQueue : assigned) { + if (!this.assignedMessageQueueState.containsKey(messageQueue)) { + MessageQueueStat messageQueueStat; + if (rebalanceImpl != null && rebalanceImpl.getProcessQueueTable().get(messageQueue) != null) { + System.out.printf("MessageQueue-%s is added %n", messageQueue); + messageQueueStat = new MessageQueueStat(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue)); + } else { + ProcessQueue processQueue = new ProcessQueue(); + messageQueueStat = new MessageQueueStat(messageQueue, processQueue); } + this.assignedMessageQueueState.put(messageQueue, messageQueueStat); } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 74cf644b98..07ef1cfa8e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -37,15 +37,12 @@ import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.hook.ConsumeMessageContext; -import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; @@ -55,15 +52,10 @@ import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; -import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.ExpressionType; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; - -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.NamespaceUtil; @@ -74,7 +66,6 @@ import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; public class DefaultLitePullConsumerImpl implements MQConsumerInner { @@ -85,8 +76,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private final RPCHook rpcHook; - private final ArrayList consumeMessageHookList = new ArrayList(); - private final ArrayList filterMessageHookList = new ArrayList(); private volatile ServiceState serviceState = ServiceState.CREATE_JUST; @@ -122,6 +111,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { * Delay some time when suspend pull service */ private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; + /** + * Delay some time when no new message + */ + private static final long PULL_TIME_DELAY_MILLS_WHEN_NO_NEW_MSG = 0; private DefaultLitePullConsumer defaultLitePullConsumer; @@ -143,10 +136,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private long nextAutoCommitDeadline = -1L; public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { - this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; - } private void checkServiceState() { @@ -162,8 +153,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } private void updateAssignedMessageQueue(String topic, Set assignedMessageQueue) { - this.assignedMessageQueue.updateAssignedMessageQueue(assignedMessageQueue); - updatePullTask(topic, assignedMessageQueue); + this.assignedMessageQueue.updateAssignedMessageQueue(topic, assignedMessageQueue); } private void updatePullTask(String topic, Set mqNewSet) { @@ -187,9 +177,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { switch (messageModel) { case BROADCASTING: updateAssignedMessageQueue(topic, mqAll); + updatePullTask(topic, mqAll); break; case CLUSTERING: updateAssignedMessageQueue(topic, mqDivided); + updatePullTask(topic, mqDivided); break; default: break; @@ -356,13 +348,16 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private void copySubscription() throws MQClientException { try { - Set registerTopics = this.defaultLitePullConsumer.getRegisterTopics(); - if (registerTopics != null) { - for (final String topic : registerTopics) { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), - topic, SubscriptionData.SUB_ALL); - this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); - } + switch (this.defaultLitePullConsumer.getMessageModel()) { + case BROADCASTING: + break; + case CLUSTERING: + /* + * Retry topic support in the future. + */ + break; + default: + break; } } catch (Exception e) { throw new MQClientException("subscription exception", e); @@ -421,7 +416,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public synchronized void unsubscribe(final String topic) { this.rebalanceImpl.getSubscriptionInner().remove(topic); - //can be delete removePullTaskCallback(topic); assignedMessageQueue.removeAssignedMessageQueue(topic); } @@ -484,8 +478,13 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } public synchronized void seek(MessageQueue messageQueue, long offset) throws MQClientException { - if (!assignedMessageQueue.messageQueues().contains(messageQueue)) - throw new MQClientException("The message queue is not in assigned list, message queue: " + messageQueue, null); + if (!assignedMessageQueue.messageQueues().contains(messageQueue)) { + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + throw new MQClientException("The message queue is not in assigned list, may be rebalancing, message queue: " + messageQueue, null); + } else { + throw new MQClientException("The message queue is not in assigned list, message queue: " + messageQueue, null); + } + } long minOffset = minOffset(messageQueue); long maxOffset = maxOffset(messageQueue); if (offset < minOffset || offset > maxOffset) @@ -552,6 +551,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } } + if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) + offsetStore.persistAll(assignedMessageQueue.messageQueues()); } catch (Exception e) { log.error("An error occurred when update consume offset synchronously.", e); } @@ -570,6 +571,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } } + if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) + offsetStore.persistAll(assignedMessageQueue.messageQueues()); } catch (Exception e) { log.error("An error occurred when update consume offset Automatically."); } @@ -605,6 +608,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { offset = assignedMessageQueue.getPullOffset(remoteQueue); if (offset == -1) { offset = fetchConsumeOffset(remoteQueue, false); + if (offset == -1 && defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { + offset = 0; + } assignedMessageQueue.updatePullOffset(remoteQueue, offset); assignedMessageQueue.updateConsumeOffset(remoteQueue, offset); } @@ -706,6 +712,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { case OFFSET_ILLEGAL: log.warn("the pull request offset illegal, {}", pullResult.toString()); break; + case NO_NEW_MSG: + pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_NO_NEW_MSG; + break; default: break; } @@ -745,7 +754,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); - return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout); } private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums) @@ -756,7 +765,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector); - return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, false, timeout); + return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout); } private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression) @@ -830,43 +839,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); //If namespace not null , reset Topic without namespace. this.resetTopic(pullResult.getMsgFoundList()); - if (!this.consumeMessageHookList.isEmpty()) { - ConsumeMessageContext consumeMessageContext = null; - consumeMessageContext = new ConsumeMessageContext(); - consumeMessageContext.setNamespace(defaultLitePullConsumer.getNamespace()); - consumeMessageContext.setConsumerGroup(this.groupName()); - consumeMessageContext.setMq(mq); - consumeMessageContext.setMsgList(pullResult.getMsgFoundList()); - consumeMessageContext.setSuccess(false); - this.executeHookBefore(consumeMessageContext); - consumeMessageContext.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); - consumeMessageContext.setSuccess(true); - this.executeHookAfter(consumeMessageContext); - } return pullResult; } - private void executeHookBefore(final ConsumeMessageContext context) { - if (!this.consumeMessageHookList.isEmpty()) { - for (ConsumeMessageHook hook : this.consumeMessageHookList) { - try { - hook.consumeMessageBefore(context); - } catch (Throwable ignored) { - } - } - } - } - - private void executeHookAfter(final ConsumeMessageContext context) { - if (!this.consumeMessageHookList.isEmpty()) { - for (ConsumeMessageHook hook : this.consumeMessageHookList) { - try { - hook.consumeMessageAfter(context); - } catch (Throwable ignored) { - } - } - } - } public void resetTopic(List msgList) { if (null == msgList || msgList.size() == 0) { @@ -920,25 +895,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { @Override public Set subscriptions() { - Set result = new HashSet(); + Set subSet = new HashSet(); - Set topics = this.defaultLitePullConsumer.getRegisterTopics(); - if (topics != null) { - synchronized (topics) { - for (String t : topics) { - SubscriptionData ms = null; - try { - ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL); - } catch (Exception e) { - log.error("parse subscription error", e); - } - ms.setSubVersion(0L); - result.add(ms); - } - } - } + subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); - return result; + return subSet; } @Override @@ -1000,41 +961,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return info; } - private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) - throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - sendMessageBack(msg, delayLevel, brokerName, this.defaultLitePullConsumer.getConsumerGroup()); - } - - private void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName, String consumerGroup) - throws RemotingException, MQBrokerException, InterruptedException, MQClientException { - try { - String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) - : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); - - if (UtilAll.isBlank(consumerGroup)) { - consumerGroup = this.defaultLitePullConsumer.getConsumerGroup(); - } - - this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, consumerGroup, delayLevel, 3000, - this.defaultLitePullConsumer.getMaxReconsumeTimes()); - } catch (Exception e) { - log.error("sendMessageBack Exception, " + this.defaultLitePullConsumer.getConsumerGroup(), e); - - Message newMsg = new Message(MixAll.getRetryTopic(this.defaultLitePullConsumer.getConsumerGroup()), msg.getBody()); - String originMsgId = MessageAccessor.getOriginMessageId(msg); - MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); - newMsg.setFlag(msg.getFlag()); - MessageAccessor.setProperties(newMsg, msg.getProperties()); - MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); - MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); - MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(this.defaultLitePullConsumer.getMaxReconsumeTimes())); - newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); - this.mQClientFactory.getDefaultMQProducer().send(newMsg); - } finally { - msg.setTopic(NamespaceUtil.withoutNamespace(msg.getTopic(), this.defaultLitePullConsumer.getNamespace())); - } - } - private void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { this.offsetStore.updateConsumeOffsetToBroker(mq, offset, isOneway); @@ -1044,6 +970,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return offsetStore; } + public void registerFilterMessageHook(final FilterMessageHook hook) { + this.filterMessageHookList.add(hook); + log.info("register FilterMessageHook Hook, {}", hook.hookName()); + } + public DefaultLitePullConsumer getDefaultLitePullConsumer() { return defaultLitePullConsumer; } diff --git a/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java b/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java index fb1f9bbde7..28e02341c6 100644 --- a/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/broadcast/PushConsumer.java @@ -50,4 +50,4 @@ public class PushConsumer { consumer.start(); System.out.printf("Broadcast Consumer Started.%n"); } -} +} \ No newline at end of file From c6096a1443f4ee53472b5a93ec68b024fd61f311 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Wed, 14 Aug 2019 10:39:36 +0800 Subject: [PATCH 08/14] Polish lite pull consumer (#1381) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle * polish LiteMQPullConsumer * add flow control and polish commit logic * fix bug * polish code * fix commit consumed offset back * refactor litePullConsumer * development save * development save * Refactor DefaultLitePullConsumer and DefaultLitePullConsumerImpl. * Polish lite pull consumer * polish lite pull consumer * polish lite pull consumer * fix seek * fix seek function * polish lite pull consumer * add apache header * add test * polish test * Make broadcast model work for litePullConsumer * Revert example/broadcast/PushConsumer.java * Add delay time when no new message * Enable long polling mode * Fix subscribe bug when rebalance * Delete useless consumeMessageHook * Implement TopicMessageQueueChangeListener interface * Lite pull consumer support namespace * Make sql92 filter work --- .../apache/rocketmq/client/ClientConfig.java | 18 +- .../consumer/DefaultLitePullConsumer.java | 25 +- .../client/consumer/LitePullConsumer.java | 9 +- .../TopicMessageQueueChangeListener.java | 30 +++ .../impl/consumer/AssignedMessageQueue.java | 2 - .../consumer/DefaultLitePullConsumerImpl.java | 232 +++++++++++------- 6 files changed, 212 insertions(+), 104 deletions(-) create mode 100644 client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index 87c01a5b30..c3e4efa252 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -16,7 +16,9 @@ */ package org.apache.rocketmq.client; +import java.util.Collection; import java.util.HashSet; +import java.util.Iterator; import java.util.Set; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; @@ -95,7 +97,6 @@ public class ClientConfig { } } - public String withNamespace(String resource) { return NamespaceUtil.wrapNamespace(this.getNamespace(), resource); } @@ -124,9 +125,21 @@ public class ClientConfig { if (StringUtils.isEmpty(this.getNamespace())) { return queue; } - return new MessageQueue(withNamespace(queue.getTopic()), queue.getBrokerName(), queue.getQueueId()); } + + public Collection queuesWithNamespace(Collection queues) { + if (StringUtils.isEmpty(this.getNamespace())) { + return queues; + } + Iterator iter = queues.iterator(); + while (iter.hasNext()) { + MessageQueue queue = iter.next(); + queue.setTopic(withNamespace(queue.getTopic())); + } + return queues; + } + public void resetClientConfig(final ClientConfig cc) { this.namesrvAddr = cc.namesrvAddr; this.clientIP = cc.clientIP; @@ -170,6 +183,7 @@ public class ClientConfig { /** * Domain name mode access way does not support the delimiter(;), and only one domain name can be set. + * * @param namesrvAddr name server address */ public void setNamesrvAddr(String namesrvAddr) { diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 1858fa1780..543e9cffb8 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -176,17 +176,22 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void subscribe(String topic, String subExpression) throws MQClientException { - this.defaultLitePullConsumerImpl.subscribe(topic, subExpression); + this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression); + } + + @Override + public void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { + this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), messageSelector); } @Override public void unsubscribe(String topic) { - this.defaultLitePullConsumerImpl.unsubscribe(topic); + this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic)); } @Override public void assign(Collection messageQueues) { - defaultLitePullConsumerImpl.assign(messageQueues); + defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues)); } @Override @@ -201,17 +206,17 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public void seek(MessageQueue messageQueue, long offset) throws MQClientException { - this.defaultLitePullConsumerImpl.seek(messageQueue, offset); + this.defaultLitePullConsumerImpl.seek(queueWithNamespace(messageQueue), offset); } @Override public void pause(Collection messageQueues) { - this.defaultLitePullConsumerImpl.pause(messageQueues); + this.defaultLitePullConsumerImpl.pause(queuesWithNamespace(messageQueues)); } @Override public void resume(Collection messageQueues) { - this.defaultLitePullConsumerImpl.resume(messageQueues); + this.defaultLitePullConsumerImpl.resume(queuesWithNamespace(messageQueues)); } @Override @@ -221,7 +226,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon @Override public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException { - return this.defaultLitePullConsumerImpl.searchOffset(messageQueue, timestamp); + return this.defaultLitePullConsumerImpl.searchOffset(queueWithNamespace(messageQueue), timestamp); + } + + public void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException { + this.defaultLitePullConsumerImpl.registerTopicMessageQueueChangeListener(withNamespace(topic), topicMessageQueueChangeListener); } @Override @@ -390,5 +400,4 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setPullDelayTimeMills(long pullDelayTimeMills) { this.pullDelayTimeMills = pullDelayTimeMills; } - } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java index ece08af61b..87b9dd354b 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -36,13 +36,20 @@ public interface LitePullConsumer { void shutdown(); /** - * Subscribe some topic + * Subscribe some topic with subExpression * * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if * null or * expression,meaning subscribe all */ void subscribe(final String topic, final String subExpression) throws MQClientException; + /** + * Subscribe some topic with selector. + * + * @param selector message selector({@link MessageSelector}), can be null. + */ + void subscribe(final String topic, final MessageSelector selector) throws MQClientException; + /** * Unsubscribe consumption some topic * diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java b/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java new file mode 100644 index 0000000000..fa6fd134e2 --- /dev/null +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/TopicMessageQueueChangeListener.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.client.consumer; + +import java.util.Set; +import org.apache.rocketmq.common.message.MessageQueue; + +public interface TopicMessageQueueChangeListener { + /** + * This method will be invoked in the condition of queue numbers changed, These scenarios occur when the topic is + * expanded or shrunk. + * + * @param messageQueues + */ + void onChanged(String topic, Set messageQueues); +} diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index b21fd01e13..8dcaa30e98 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -138,7 +138,6 @@ public class AssignedMessageQueue { Map.Entry next = it.next(); if (next.getKey().getTopic().equals(topic)) { if (!assigned.contains(next.getKey())) { - System.out.printf("MessageQueue-%s is removed %n", next.getKey()); next.getValue().getProcessQueue().setDropped(true); it.remove(); } @@ -167,7 +166,6 @@ public class AssignedMessageQueue { if (!this.assignedMessageQueueState.containsKey(messageQueue)) { MessageQueueStat messageQueueStat; if (rebalanceImpl != null && rebalanceImpl.getProcessQueueTable().get(messageQueue) != null) { - System.out.printf("MessageQueue-%s is added %n", messageQueue); messageQueueStat = new MessageQueueStat(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue)); } else { ProcessQueue processQueue = new ProcessQueue(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 07ef1cfa8e..e17aae65ac 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.client.impl.consumer; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; import java.util.Iterator; @@ -28,12 +29,16 @@ import java.util.Properties; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.client.consumer.TopicMessageQueueChangeListener; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.MessageQueueListener; import org.apache.rocketmq.client.consumer.PullResult; @@ -127,6 +132,12 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + private final ScheduledExecutorService scheduledExecutorService; + + private Map topicMessageQueueChangeListenerMap = new HashMap(); + + private Map> messageQueuesForTopic = new HashMap>(); + private long consumeRequestFlowControlTimes = 0L; private long queueFlowControlTimes = 0L; @@ -138,6 +149,16 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( + this.defaultLitePullConsumer.getPullThreadNumbers(), + new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) + ); + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "MonitorMessageQueueChangeThread"); + } + }); } private void checkServiceState() { @@ -266,12 +287,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { mQClientFactory.start(); - final String group = this.defaultLitePullConsumer.getConsumerGroup(); - - this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - this.defaultLitePullConsumer.getPullThreadNumbers(), - new ThreadFactoryImpl("PullMsgThread-" + group) - ); if (subscriptionType == SubscriptionType.SUBSCRIBE) { updateTopicSubscribeInfoWhenSubscriptionChanged(); } @@ -279,8 +294,25 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { updateAssignPullTask(assignedMessageQueue.messageQueues()); } + scheduledExecutorService.scheduleAtFixedRate( + new Runnable() { + @Override + public void run() { + try { + fetchTopicMessageQueuesAndCompare(); + } catch (Exception e) { + log.error("ScheduledTask fetchMessageQueuesAndCompare exception", e); + } + } + }, 1000 * 20, 1000 * 30, TimeUnit.MILLISECONDS); + log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup()); this.serviceState = ServiceState.RUNNING; + for (String topic : topicMessageQueueChangeListenerMap.keySet()) { + Set messageQueues = fetchMessageQueues(topic); + messageQueuesForTopic.put(topic, messageQueues); + } + this.mQClientFactory.checkClientInBroker(); break; case RUNNING: case START_FAILED: @@ -339,7 +371,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } - } public PullAPIWrapper getPullAPIWrapper() { @@ -353,7 +384,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { break; case CLUSTERING: /* - * Retry topic support in the future. + * Retry topic will be support in the future. */ break; default: @@ -410,7 +441,28 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { updateTopicSubscribeInfoWhenSubscriptionChanged(); } } catch (Exception e) { - throw new MQClientException("subscription exception", e); + throw new MQClientException("subscribe exception", e); + } + } + + public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { + try { + setSubscriptionType(SubscriptionType.SUBSCRIBE); + if (messageSelector == null) { + subscribe(topic, SubscriptionData.SUB_ALL); + return; + } + SubscriptionData subscriptionData = FilterAPI.build(topic, + messageSelector.getExpression(), messageSelector.getExpressionType()); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + this.defaultLitePullConsumer.setMessageQueueListener(new MessageQueueListenerImpl()); + assignedMessageQueue.setRebalanceImpl(this.rebalanceImpl); + if (serviceState == ServiceState.RUNNING) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + } catch (Exception e) { + throw new MQClientException("subscribe exception", e); } } @@ -421,6 +473,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } public synchronized void assign(Collection messageQueues) { + if (messageQueues == null || messageQueues.isEmpty()) { + throw new IllegalArgumentException("Message queues can not be null or empty."); + } setSubscriptionType(SubscriptionType.ASSIGN); assignedMessageQueue.updateAssignedMessageQueue(messageQueues); if (serviceState == ServiceState.RUNNING) { @@ -461,6 +516,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { List messages = consumeRequest.getMessageExts(); long offset = consumeRequest.getProcessQueue().removeMessage(messages); assignedMessageQueue.updateConsumeOffset(consumeRequest.getMessageQueue(), offset); + //If namespace not null , reset Topic without namespace. + this.resetTopic(messages); return messages; } } catch (InterruptedException ignore) { @@ -587,8 +644,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private void submitConsumeRequest(ConsumeRequest consumeRequest) { try { consumeRequestCache.put(consumeRequest); - } catch (InterruptedException ex) { - log.error("Submit consumeRequest error", ex); + } catch (InterruptedException e) { + log.error("Submit consumeRequest error", e); } } @@ -649,14 +706,14 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); if (processQueue == null && processQueue.isDropped()) { - log.info("the message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); + log.info("The message queue not be able to poll, because it's dropped. group={}, messageQueue={}", defaultLitePullConsumer.getConsumerGroup(), this.messageQueue); return; } if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchNums() > defaultLitePullConsumer.getPullThresholdForAll()) { scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); if ((consumeRequestFlowControlTimes++ % 1000) == 0) - log.warn("the consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); + log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); return; } @@ -667,7 +724,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( - "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + "The cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", defaultLitePullConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); } return; @@ -677,7 +734,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( - "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", + "The cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, flowControlTimes={}", defaultLitePullConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, queueFlowControlTimes); } return; @@ -687,21 +744,27 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { log.warn( - "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", + "The queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), queueMaxSpanFlowControlTimes); } return; } - String subExpression = null; - if (subscriptionType == SubscriptionType.SUBSCRIBE) { - String topic = this.messageQueue.getTopic(); - subExpression = rebalanceImpl.getSubscriptionInner().get(topic).getSubString(); - } long offset = nextPullOffset(messageQueue); long pullDelayTimeMills = defaultLitePullConsumer.getPullDelayTimeMills(); try { - PullResult pullResult = pull(messageQueue, subExpression, offset, nextPullBatchNums()); + + SubscriptionData subscriptionData; + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + String topic = this.messageQueue.getTopic(); + subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); + } else{ + String topic = this.messageQueue.getTopic(); + subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), + topic, SubscriptionData.SUB_ALL); + } + + PullResult pullResult = pull(messageQueue, subscriptionData, offset, nextPullBatchNums()); switch (pullResult.getPullStatus()) { case FOUND: if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty()) { @@ -710,7 +773,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } break; case OFFSET_ILLEGAL: - log.warn("the pull request offset illegal, {}", pullResult.toString()); + log.warn("The pull request offset illegal, {}", pullResult.toString()); break; case NO_NEW_MSG: pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_NO_NEW_MSG; @@ -721,7 +784,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { updatePullOffset(messageQueue, pullResult.getNextBeginOffset()); } catch (Throwable e) { pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION; - e.printStackTrace(); log.error("An error occurred in pull message process.", e); } @@ -746,58 +808,16 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return pull(mq, subExpression, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); + return pull(mq, subscriptionData, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); } - private PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums, long timeout) + private PullResult pull(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, long timeout) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - SubscriptionData subscriptionData = getSubscriptionData(mq, subExpression); return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout); } - private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - return pull(mq, messageSelector, offset, maxNums, this.defaultLitePullConsumer.getConsumerPullTimeoutMillis()); - } - - private PullResult pull(MessageQueue mq, MessageSelector messageSelector, long offset, int maxNums, long timeout) - throws MQClientException, RemotingException, MQBrokerException, InterruptedException { - SubscriptionData subscriptionData = getSubscriptionData(mq, messageSelector); - return this.pullSyncImpl(mq, subscriptionData, offset, maxNums, true, timeout); - } - - private SubscriptionData getSubscriptionData(MessageQueue mq, String subExpression) - throws MQClientException { - - if (null == mq) { - throw new MQClientException("mq is null", null); - } - - try { - return FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), - mq.getTopic(), subExpression); - } catch (Exception e) { - throw new MQClientException("parse subscription error", e); - } - } - - private SubscriptionData getSubscriptionData(MessageQueue mq, MessageSelector messageSelector) - throws MQClientException { - - if (null == mq) { - throw new MQClientException("mq is null", null); - } - - try { - return FilterAPI.build(mq.getTopic(), - messageSelector.getExpression(), messageSelector.getExpressionType()); - } catch (Exception e) { - throw new MQClientException("parse subscription error", e); - } - } - private PullResult pullSyncImpl(MessageQueue mq, SubscriptionData subscriptionData, long offset, int maxNums, boolean block, long timeout) @@ -815,8 +835,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { throw new MQClientException("maxNums <= 0", null); } - this.subscriptionAutomatically(mq.getTopic()); - int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); long timeoutMillis = block ? this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() : timeout; @@ -837,13 +855,10 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null ); this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); - //If namespace not null , reset Topic without namespace. - this.resetTopic(pullResult.getMsgFoundList()); return pullResult; } - - public void resetTopic(List msgList) { + private void resetTopic(List msgList) { if (null == msgList || msgList.size() == 0) { return; } @@ -857,17 +872,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } - public void subscriptionAutomatically(final String topic) { - if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) { - try { - SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultLitePullConsumer.getConsumerGroup(), - topic, SubscriptionData.SUB_ALL); - this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData); - } catch (Exception ignore) { - } - } - } - public void updateConsumeOffset(MessageQueue mq, long offset) { checkServiceState(); this.offsetStore.updateOffset(mq, offset, false); @@ -981,13 +985,59 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public Set fetchMessageQueues(String topic) throws MQClientException { checkServiceState(); - // check if has info in memory, otherwise invoke api. - Set result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); - if (null == result) { - result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + Set result = this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + return parseMessageQueues(result); + } + + private synchronized void fetchTopicMessageQueuesAndCompare() throws MQClientException { + for (Map.Entry entry : topicMessageQueueChangeListenerMap.entrySet()) { + String topic = entry.getKey(); + TopicMessageQueueChangeListener topicMessageQueueChangeListener = entry.getValue(); + Set oldMessageQueues = messageQueuesForTopic.get(topic); + Set newMessageQueues = fetchMessageQueues(topic); + boolean isChanged = !isSetEqual(newMessageQueues, oldMessageQueues); + if (isChanged) { + messageQueuesForTopic.put(topic, newMessageQueues); + if (topicMessageQueueChangeListener != null) { + topicMessageQueueChangeListener.onChanged(topic, newMessageQueues); + } + } + } + } + + private boolean isSetEqual(Set set1, Set set2) { + if (set1 == null && set2 == null) { + return true; } - return parseMessageQueues(result); + if (set1 == null || set2 == null || set1.size() != set2.size() + || set1.size() == 0 || set2.size() == 0) { + return false; + } + + Iterator iter = set2.iterator(); + boolean isEqual = true; + while (iter.hasNext()) { + if (!set1.contains(iter.next())) { + isEqual = false; + } + } + return isEqual; + } + + public synchronized void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener listener) throws MQClientException { + if (topic == null || listener == null) { + throw new MQClientException("Topic or listener is null", null); + } + if (topicMessageQueueChangeListenerMap.containsKey(topic)) { + log.warn("Topic {} had been registered, new listener will overwrite the old one", topic); + } + topicMessageQueueChangeListenerMap.put(topic, listener); + if (this.serviceState == ServiceState.RUNNING) { + Set messageQueues = fetchMessageQueues(topic); + messageQueuesForTopic.put(topic, messageQueues); + } } private Set parseMessageQueues(Set queueSet) { From ddb07ab13c70292b7e63ef895da0bdf1c11f3eda Mon Sep 17 00:00:00 2001 From: Heng Du Date: Wed, 14 Aug 2019 17:27:31 +0800 Subject: [PATCH 09/14] Deprecated the DefaulteMQPull consumer (#1387) --- .../rocketmq/client/consumer/DefaultMQPullConsumer.java | 5 ++++- .../client/consumer/MQPullConsumerScheduleService.java | 4 +++- .../client/impl/consumer/DefaultLitePullConsumerImpl.java | 2 +- .../{LitePullConsumerTest.java => LitePullConsumer.java} | 6 +++--- 4 files changed, 11 insertions(+), 6 deletions(-) rename example/src/main/java/org/apache/rocketmq/example/simple/{LitePullConsumerTest.java => LitePullConsumer.java} (94%) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index 3fa3af2402..e6d3f70484 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -35,8 +35,11 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.exception.RemotingException; /** - * Default pulling consumer + * Default pulling consumer. + * This Consumer will be removed in 2022, and a better implementation {@link DefaultLitePullConsumer} is recommend to use + * in the scenario of actively pulling messages. */ +@Deprecated public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer { protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java index 685f4c8d2d..4dcc24b1b7 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -31,7 +31,9 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; /** - * Schedule service for pull consumer + * Schedule service for pull consumer. + * This Consumer will be removed in 2022, and a better implementation {@link + * DefaultLitePullConsumer} is recommend to use in the scenario of actively pulling messages. */ public class MQPullConsumerScheduleService { private final InternalLogger log = ClientLogger.getLog(); diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index e17aae65ac..8ff2b39c4d 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -758,7 +758,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { if (subscriptionType == SubscriptionType.SUBSCRIBE) { String topic = this.messageQueue.getTopic(); subscriptionData = rebalanceImpl.getSubscriptionInner().get(topic); - } else{ + } else { String topic = this.messageQueue.getTopic(); subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), topic, SubscriptionData.SUB_ALL); diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java similarity index 94% rename from example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java rename to example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java index 0430465cf5..b0a2b596ad 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerTest.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java @@ -24,12 +24,12 @@ import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -public class LitePullConsumerTest { +public class LitePullConsumer { public static void main(String[] args) throws Exception { - DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("test"); + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("please_rename_unique_group_name"); litePullConsumer.setAutoCommit(false); litePullConsumer.start(); - Collection mqSet = litePullConsumer.fetchMessageQueues("test400"); + Collection mqSet = litePullConsumer.fetchMessageQueues("TopicTest"); List list = new ArrayList<>(mqSet); Collection assginMq = Collections.singletonList(list.get(0)); litePullConsumer.assign(assginMq); From cfca4401d5a0018f506aa655c452fb6a98ab5a76 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Mon, 19 Aug 2019 14:57:16 +0800 Subject: [PATCH 10/14] Polish LitePullConsumer (#1395) * fix unsubscribe code * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * fix commit consumed offset * polish commit consumed offset * pass checkstyle * pass checkstyle * polish LiteMQPullConsumer * add flow control and polish commit logic * fix bug * polish code * fix commit consumed offset back * refactor litePullConsumer * development save * development save * Refactor DefaultLitePullConsumer and DefaultLitePullConsumerImpl. * Polish lite pull consumer * polish lite pull consumer * polish lite pull consumer * fix seek * fix seek function * polish lite pull consumer * add apache header * add test * polish test * Make broadcast model work for litePullConsumer * Revert example/broadcast/PushConsumer.java * Add delay time when no new message * Enable long polling mode * Fix subscribe bug when rebalance * Delete useless consumeMessageHook * Implement TopicMessageQueueChangeListener interface * Lite pull consumer support namespace * Make sql92 filter work * 1. Add javadoc for lite pull consumer. 2. Polish lite pul consumer parameters * Modify lite pull consumer example * Add apache header for LitePullConsumerAssign example. * Add unit test for lite pull consumer * Fix typo --- .../consumer/DefaultLitePullConsumer.java | 199 ++++++++++---- .../client/consumer/LitePullConsumer.java | 104 ++++++- .../impl/consumer/AssignedMessageQueue.java | 94 +++---- .../consumer/DefaultLitePullConsumerImpl.java | 74 ++--- .../consumer/DefaultLitePullConsumerTest.java | 253 +++++++++++------- ...sumer.java => LitePullConsumerAssign.java} | 30 ++- .../simple/LitePullConsumerSubscribe.java | 42 +++ 7 files changed, 558 insertions(+), 238 deletions(-) rename example/src/main/java/org/apache/rocketmq/example/simple/{LitePullConsumer.java => LitePullConsumerAssign.java} (70%) create mode 100644 example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 543e9cffb8..33d9e5fc9f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -77,28 +77,25 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private boolean unitMode = false; - private int maxReconsumeTimes = 16; /** - * Maximum amount of time in minutes a message may block the consuming thread. - */ - private long consumeTimeout = 15; - - /** - * Is auto commit offset + * The flag for auto commit offset */ private boolean autoCommit = true; - private int pullThreadNumbers = 20; + /** + * Pull thread number + */ + private int pullThreadNums = 20; /** - * Maximum commit offset interval time in seconds. + * Maximum commit offset interval time in milliseconds. */ - private long autoCommitInterval = 5; + private long autoCommitIntervalMillis = 5 * 1000; /** * Maximum number of messages pulled each time. */ - private int pullBatchNums = 10; + private int pullBatchSize = 10; /** * Flow control threshold for consume request, each consumer will cache at most 10000 consume requests by default. @@ -132,22 +129,36 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon private long pollTimeoutMillis = 1000 * 5; /** - * Message pull delay in milliseconds + * Default constructor. */ - private long pullDelayTimeMills = 0; - public DefaultLitePullConsumer() { this(null, MixAll.DEFAULT_CONSUMER_GROUP, null); } + /** + * Constructor specifying consumer group. + * + * @param consumerGroup Consumer group. + */ public DefaultLitePullConsumer(final String consumerGroup) { this(null, consumerGroup, null); } + /** + * Constructor specifying RPC hook. + * + * @param rpcHook RPC hook to execute before each remoting command. + */ public DefaultLitePullConsumer(RPCHook rpcHook) { this(null, MixAll.DEFAULT_CONSUMER_GROUP, rpcHook); } + /** + * Constructor specifying consumer group, RPC hook + * + * @param consumerGroup Consumer group. + * @param rpcHook RPC hook to execute before each remoting command. + */ public DefaultLitePullConsumer(final String consumerGroup, RPCHook rpcHook) { this(null, consumerGroup, rpcHook); } @@ -164,121 +175,229 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook); } + /** + * Start the consumer + */ @Override public void start() throws MQClientException { this.defaultLitePullConsumerImpl.start(); } + /** + * Shutdown the consumer + */ @Override public void shutdown() { this.defaultLitePullConsumerImpl.shutdown(); } + /** + * Subscribe some topic with subExpression + * + * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if + * null or * expression,meaning subscribe all + * @throws MQClientException if there is any client error. + */ @Override public void subscribe(String topic, String subExpression) throws MQClientException { this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression); } + /** + * Subscribe some topic with selector. + * + * @param messageSelector message selector({@link MessageSelector}), can be null. + * @throws MQClientException if there is any client error. + */ @Override public void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), messageSelector); } + /** + * Unsubscribe consumption some topic + * + * @param topic Message topic that needs to be unsubscribe. + */ @Override public void unsubscribe(String topic) { this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic)); } + /** + * Manually assign a list of message queues to this consumer. This interface does not allow for incremental + * assignment and will replace the previous assignment (if there is one). + * + * @param messageQueues Message queues that needs to be assigned. + */ @Override public void assign(Collection messageQueues) { defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues)); } + /** + * Fetch data for the topics or partitions specified using assign API + * + * @return list of message, can be null. + */ @Override public List poll() { return defaultLitePullConsumerImpl.poll(this.getPollTimeoutMillis()); } + /** + * Fetch data for the topics or partitions specified using assign API + * + * @param timeout The amount time, in milliseconds, spent waiting in poll if data is not available. Must not be + * negative + * @return list of message, can be null. + */ @Override public List poll(long timeout) { return defaultLitePullConsumerImpl.poll(timeout); } + /** + * Overrides the fetch offsets that the consumer will use on the next poll. If this API is invoked for the same + * message queue more than once, the latest offset will be used on the next poll(). Note that you may lose data if + * this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + * @param offset + */ @Override public void seek(MessageQueue messageQueue, long offset) throws MQClientException { this.defaultLitePullConsumerImpl.seek(queueWithNamespace(messageQueue), offset); } + /** + * Suspend pulling from the requested message queues. + * + * Because of the implementation of pre-pull, fetch data in {@link #poll()} will not stop immediately until the + * messages of the requested message queues drain. + * + * Note that this method does not affect message queue subscription. In particular, it does not cause a group + * rebalance. + * + * @param messageQueues Message queues that needs to be paused. + */ @Override public void pause(Collection messageQueues) { this.defaultLitePullConsumerImpl.pause(queuesWithNamespace(messageQueues)); } + /** + * Resume specified message queues which have been paused with {@link #pause(Collection)}. + * + * @param messageQueues Message queues that needs to be resumed. + */ @Override public void resume(Collection messageQueues) { this.defaultLitePullConsumerImpl.resume(queuesWithNamespace(messageQueues)); } + /** + * Get metadata about the message queues for a given topic. + * + * @param topic The topic that need to get metadata. + * @return collection of message queues + * @throws MQClientException if there is any client error. + */ @Override public Collection fetchMessageQueues(String topic) throws MQClientException { return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic)); } + /** + * Look up the offsets for the given message queue by timestamp. The returned offset for each message queue is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message + * queue. + * + * @param messageQueue Message queues that needs to get offset by timestamp. + * @param timestamp + * @return offset + * @throws MQClientException if there is any client error. + */ @Override public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException { return this.defaultLitePullConsumerImpl.searchOffset(queueWithNamespace(messageQueue), timestamp); } + /** + * Register a callback for sensing topic metadata changes. + * + * @param topic The topic that need to monitor. + * @param topicMessageQueueChangeListener Callback when topic metadata changes. + * @throws MQClientException if there is any client error. + */ + @Override public void registerTopicMessageQueueChangeListener(String topic, TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException { this.defaultLitePullConsumerImpl.registerTopicMessageQueueChangeListener(withNamespace(topic), topicMessageQueueChangeListener); } + /** + * Manually commit consume offset. + */ @Override public void commitSync() { this.defaultLitePullConsumerImpl.commitSync(); } + /** + * Get the last committed offset for the given message queue. + * + * @param messageQueue + * @return offset, if offset equals -1 means no offset in broker. + * @throws MQClientException if there is any client error. + */ + @Override + public Long committed(MessageQueue messageQueue) throws MQClientException { + return this.defaultLitePullConsumerImpl.committed(messageQueue); + } + + /** + * Whether to enable auto-commit consume offset. + * + * @return true if enable auto-commit, false if disable auto-commit. + */ @Override public boolean isAutoCommit() { return autoCommit; } + /** + * Set whether to enable auto-commit consume offset. + * + * @param autoCommit Whether to enable auto-commit. + */ @Override public void setAutoCommit(boolean autoCommit) { this.autoCommit = autoCommit; } - public long getConsumeTimeout() { - return consumeTimeout; + public int getPullThreadNums() { + return pullThreadNums; } - public void setConsumeTimeout(long consumeTimeout) { - this.consumeTimeout = consumeTimeout; + public void setPullThreadNums(int pullThreadNums) { + this.pullThreadNums = pullThreadNums; } - public int getPullThreadNumbers() { - return pullThreadNumbers; + public long getAutoCommitIntervalMillis() { + return autoCommitIntervalMillis; } - public void setPullThreadNumbers(int pullThreadNumbers) { - this.pullThreadNumbers = pullThreadNumbers; - } - - public long getAutoCommitInterval() { - return autoCommitInterval; - } - - public void setAutoCommitInterval(long autoCommitInterval) { - this.autoCommitInterval = autoCommitInterval; + public void setAutoCommitIntervalMillis(long autoCommitIntervalMillis) { + this.autoCommitIntervalMillis = autoCommitIntervalMillis; } public int getPullBatchNums() { - return pullBatchNums; + return pullBatchSize; } public void setPullBatchNums(int pullBatchNums) { - this.pullBatchNums = pullBatchNums; + this.pullBatchSize = pullBatchNums; } public long getPullThresholdForAll() { @@ -349,14 +468,6 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon this.unitMode = isUnitMode; } - public int getMaxReconsumeTimes() { - return maxReconsumeTimes; - } - - public void setMaxReconsumeTimes(final int maxReconsumeTimes) { - this.maxReconsumeTimes = maxReconsumeTimes; - } - public MessageModel getMessageModel() { return messageModel; } @@ -392,12 +503,4 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; } - - public long getPullDelayTimeMills() { - return pullDelayTimeMills; - } - - public void setPullDelayTimeMills(long pullDelayTimeMills) { - this.pullDelayTimeMills = pullDelayTimeMills; - } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java index 87b9dd354b..d6e657ff18 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/LitePullConsumer.java @@ -39,7 +39,8 @@ public interface LitePullConsumer { * Subscribe some topic with subExpression * * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if - * null or * expression,meaning subscribe all + * null or * expression,meaning subscribe all + * @throws MQClientException if there is any client error. */ void subscribe(final String topic, final String subExpression) throws MQClientException; @@ -47,37 +48,128 @@ public interface LitePullConsumer { * Subscribe some topic with selector. * * @param selector message selector({@link MessageSelector}), can be null. + * @throws MQClientException if there is any client error. */ void subscribe(final String topic, final MessageSelector selector) throws MQClientException; /** * Unsubscribe consumption some topic * - * @param topic message topic + * @param topic Message topic that needs to be unsubscribe. */ void unsubscribe(final String topic); + /** + * Manually assign a list of message queues to this consumer. This interface does not allow for incremental + * assignment and will replace the previous assignment (if there is one). + * + * @param messageQueues Message queues that needs to be assigned. + */ void assign(Collection messageQueues); + /** + * Fetch data for the topics or partitions specified using assign API + * + * @return list of message, can be null. + */ List poll(); + /** + * Fetch data for the topics or partitions specified using assign API + * + * @param timeout The amount time, in milliseconds, spent waiting in poll if data is not available. Must not be + * negative + * @return list of message, can be null. + */ List poll(long timeout); + /** + * Overrides the fetch offsets that the consumer will use on the next poll. If this API is invoked for the same + * message queue more than once, the latest offset will be used on the next poll(). Note that you may lose data if + * this API is arbitrarily used in the middle of consumption. + * + * @param messageQueue + * @param offset + */ void seek(MessageQueue messageQueue, long offset) throws MQClientException; + /** + * Suspend pulling from the requested message queues. + * + * Because of the implementation of pre-pull, fetch data in {@link #poll()} will not stop immediately until the + * messages of the requested message queues drain. + * + * Note that this method does not affect message queue subscription. In particular, it does not cause a group + * rebalance. + * + * @param messageQueues Message queues that needs to be paused. + */ void pause(Collection messageQueues); - boolean isAutoCommit(); - - void setAutoCommit(boolean autoCommit); - + /** + * Resume specified message queues which have been paused with {@link #pause(Collection)}. + * + * @param messageQueues Message queues that needs to be resumed. + */ void resume(Collection messageQueues); + /** + * Whether to enable auto-commit consume offset. + * + * @return true if enable auto-commit, false if disable auto-commit. + */ + boolean isAutoCommit(); + + /** + * Set whether to enable auto-commit consume offset. + * + * @param autoCommit Whether to enable auto-commit. + */ + void setAutoCommit(boolean autoCommit); + + /** + * Get metadata about the message queues for a given topic. + * + * @param topic The topic that need to get metadata. + * @return collection of message queues + * @throws MQClientException if there is any client error. + */ Collection fetchMessageQueues(String topic) throws MQClientException; + /** + * Look up the offsets for the given message queue by timestamp. The returned offset for each message queue is the + * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message + * queue. + * + * @param messageQueue Message queues that needs to get offset by timestamp. + * @param timestamp + * @return offset + * @throws MQClientException if there is any client error. + */ Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException; + /** + * Manually commit consume offset. + */ void commitSync(); + /** + * Get the last committed offset for the given message queue. + * + * @param messageQueue + * @return offset, if offset equals -1 means no offset in broker. + * @throws MQClientException if there is any client error. + */ + Long committed(MessageQueue messageQueue) throws MQClientException; + /** + * Register a callback for sensing topic metadata changes. + * + * @param topic The topic that need to monitor. + * @param topicMessageQueueChangeListener Callback when topic metadata changes, refer {@link + * TopicMessageQueueChangeListener} + * @throws MQClientException if there is any client error. + */ + void registerTopicMessageQueueChangeListener(String topic, + TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index 8dcaa30e98..609fc4d1ef 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -26,12 +26,12 @@ import org.apache.rocketmq.common.message.MessageQueue; public class AssignedMessageQueue { - private ConcurrentHashMap assignedMessageQueueState; + private ConcurrentHashMap assignedMessageQueueState; private RebalanceImpl rebalanceImpl; public AssignedMessageQueue() { - assignedMessageQueueState = new ConcurrentHashMap(); + assignedMessageQueueState = new ConcurrentHashMap(); } public void setRebalanceImpl(RebalanceImpl rebalanceImpl) { @@ -43,99 +43,99 @@ public class AssignedMessageQueue { } public boolean isPaused(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.isPaused(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.isPaused(); } return true; } public void pause(Collection messageQueues) { for (MessageQueue messageQueue : messageQueues) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { - messageQueueStat.getPausedLatch().reset(); - messageQueueStat.setPaused(true); + messageQueueState.getPausedLatch().reset(); + messageQueueState.setPaused(true); } } } public void resume(Collection messageQueueCollection) { for (MessageQueue messageQueue : messageQueueCollection) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { - messageQueueStat.setPaused(false); - messageQueueStat.getPausedLatch().reset(); + messageQueueState.setPaused(false); + messageQueueState.getPausedLatch().reset(); } } } public ProcessQueue getProcessQueue(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.getProcessQueue(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getProcessQueue(); } return null; } public long getPullOffset(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.getPullOffset(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getPullOffset(); } return -1; } public void updatePullOffset(MessageQueue messageQueue, long offset) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - messageQueueStat.setPullOffset(offset); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setPullOffset(offset); } } public long getConusmerOffset(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.getConsumeOffset(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getConsumeOffset(); } return -1; } public void updateConsumeOffset(MessageQueue messageQueue, long offset) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - messageQueueStat.setConsumeOffset(offset); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setConsumeOffset(offset); } } public void setSeekOffset(MessageQueue messageQueue, long offset) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - messageQueueStat.setSeekOffset(offset); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + messageQueueState.setSeekOffset(offset); } } public long getSeekOffset(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.getSeekOffset(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getSeekOffset(); } return -1; } public CountDownLatch2 getPausedLatch(MessageQueue messageQueue) { - MessageQueueStat messageQueueStat = assignedMessageQueueState.get(messageQueue); - if (messageQueueStat != null) { - return messageQueueStat.getPausedLatch(); + MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); + if (messageQueueState != null) { + return messageQueueState.getPausedLatch(); } return null; } public void updateAssignedMessageQueue(String topic, Collection assigned) { synchronized (this.assignedMessageQueueState) { - Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); while (it.hasNext()) { - Map.Entry next = it.next(); + Map.Entry next = it.next(); if (next.getKey().getTopic().equals(topic)) { if (!assigned.contains(next.getKey())) { next.getValue().getProcessQueue().setDropped(true); @@ -149,9 +149,9 @@ public class AssignedMessageQueue { public void updateAssignedMessageQueue(Collection assigned) { synchronized (this.assignedMessageQueueState) { - Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); while (it.hasNext()) { - Map.Entry next = it.next(); + Map.Entry next = it.next(); if (!assigned.contains(next.getKey())) { next.getValue().getProcessQueue().setDropped(true); it.remove(); @@ -164,23 +164,23 @@ public class AssignedMessageQueue { private void addAssignedMessageQueue(Collection assigned) { for (MessageQueue messageQueue : assigned) { if (!this.assignedMessageQueueState.containsKey(messageQueue)) { - MessageQueueStat messageQueueStat; + MessageQueueState messageQueueState; if (rebalanceImpl != null && rebalanceImpl.getProcessQueueTable().get(messageQueue) != null) { - messageQueueStat = new MessageQueueStat(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue)); + messageQueueState = new MessageQueueState(messageQueue, rebalanceImpl.getProcessQueueTable().get(messageQueue)); } else { ProcessQueue processQueue = new ProcessQueue(); - messageQueueStat = new MessageQueueStat(messageQueue, processQueue); + messageQueueState = new MessageQueueState(messageQueue, processQueue); } - this.assignedMessageQueueState.put(messageQueue, messageQueueStat); + this.assignedMessageQueueState.put(messageQueue, messageQueueState); } } } public void removeAssignedMessageQueue(String topic) { synchronized (this.assignedMessageQueueState) { - Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); + Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); while (it.hasNext()) { - Map.Entry next = it.next(); + Map.Entry next = it.next(); if (next.getKey().getTopic().equals(topic)) { it.remove(); } @@ -188,7 +188,7 @@ public class AssignedMessageQueue { } } - public class MessageQueueStat { + private class MessageQueueState { private MessageQueue messageQueue; private ProcessQueue processQueue; private volatile boolean paused = false; @@ -197,7 +197,7 @@ public class AssignedMessageQueue { private volatile long seekOffset = -1; private CountDownLatch2 pausedLatch = new CountDownLatch2(1); - public MessageQueueStat(MessageQueue messageQueue, ProcessQueue processQueue) { + public MessageQueueState(MessageQueue messageQueue, ProcessQueue processQueue) { this.messageQueue = messageQueue; this.processQueue = processQueue; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 8ff2b39c4d..2fb33385d5 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -97,9 +97,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { NONE, SUBSCRIBE, ASSIGN } - private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running."; + private static final String NOT_RUNNING_EXCEPTION_MESSAGE = "The consumer not running, please start it first."; - private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Cannot select two subscription types at the same time."; + private static final String SUBSCRIPTION_CONFILCT_EXCEPTION_MESSAGE = "Subscribe and assign are mutually exclusive."; /** * the type of subscription */ @@ -116,10 +116,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { * Delay some time when suspend pull service */ private static final long PULL_TIME_DELAY_MILLS_WHEN_PAUSE = 1000; - /** - * Delay some time when no new message - */ - private static final long PULL_TIME_DELAY_MILLS_WHEN_NO_NEW_MSG = 0; private DefaultLitePullConsumer defaultLitePullConsumer; @@ -150,7 +146,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor( - this.defaultLitePullConsumer.getPullThreadNumbers(), + this.defaultLitePullConsumer.getPullThreadNums(), new ThreadFactoryImpl("PullMsgThread-" + this.defaultLitePullConsumer.getConsumerGroup()) ); this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { @@ -219,11 +215,12 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { case CREATE_JUST: break; case RUNNING: - this.persistConsumerOffset(); + persistConsumerOffset(); this.mQClientFactory.unregisterConsumer(this.defaultLitePullConsumer.getConsumerGroup()); this.mQClientFactory.shutdown(); log.info("the consumer [{}] shutdown OK", this.defaultLitePullConsumer.getConsumerGroup()); scheduledThreadPoolExecutor.shutdown(); + scheduledExecutorService.shutdown(); this.serviceState = ServiceState.SHUTDOWN_ALREADY; break; case SHUTDOWN_ALREADY: @@ -430,6 +427,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public synchronized void subscribe(String topic, String subExpression) throws MQClientException { try { + if (topic == null || topic.equals("")) { + throw new IllegalArgumentException("Topic can not be null or empty."); + } setSubscriptionType(SubscriptionType.SUBSCRIBE); SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(defaultLitePullConsumer.getConsumerGroup(), topic, subExpression); @@ -447,6 +447,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { public synchronized void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { try { + if (topic == null || topic.equals("")) { + throw new IllegalArgumentException("Topic can not be null or empty."); + } setSubscriptionType(SubscriptionType.SUBSCRIBE); if (messageSelector == null) { subscribe(topic, SubscriptionData.SUB_ALL); @@ -487,7 +490,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { long now = System.currentTimeMillis(); if (now >= nextAutoCommitDeadline) { commitAll(); - nextAutoCommitDeadline = now + defaultLitePullConsumer.getAutoCommitInterval() * 1000; + nextAutoCommitDeadline = now + defaultLitePullConsumer.getAutoCommitIntervalMillis(); } } @@ -570,21 +573,21 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - public long maxOffset(MessageQueue mq) throws MQClientException { + private long maxOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); - return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + return this.mQClientFactory.getMQAdminImpl().maxOffset(messageQueue); } - public long minOffset(MessageQueue mq) throws MQClientException { + private long minOffset(MessageQueue messageQueue) throws MQClientException { checkServiceState(); - return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + return this.mQClientFactory.getMQAdminImpl().minOffset(messageQueue); } - public void removePullTaskCallback(final String topic) { + private void removePullTaskCallback(final String topic) { removePullTask(topic); } - public void removePullTask(final String topic) { + private void removePullTask(final String topic) { Iterator> it = this.taskTable.entrySet().iterator(); while (it.hasNext()) { Map.Entry next = it.next(); @@ -615,7 +618,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - public synchronized void commitAll() { + private synchronized void commitAll() { try { for (MessageQueue messageQueue : assignedMessageQueue.messageQueues()) { long consumerOffset = assignedMessageQueue.getConusmerOffset(messageQueue); @@ -635,9 +638,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private void updatePullOffset(MessageQueue remoteQueue, long nextPullOffset) { - if (assignedMessageQueue.getSeekOffset(remoteQueue) == -1) { - assignedMessageQueue.updatePullOffset(remoteQueue, nextPullOffset); + private void updatePullOffset(MessageQueue messageQueue, long nextPullOffset) { + if (assignedMessageQueue.getSeekOffset(messageQueue) == -1) { + assignedMessageQueue.updatePullOffset(messageQueue, nextPullOffset); } } @@ -649,27 +652,35 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private long fetchConsumeOffset(MessageQueue mq, boolean fromStore) { + private long fetchConsumeOffset(MessageQueue messageQueue, boolean fromStore) { checkServiceState(); - return this.offsetStore.readOffset(mq, fromStore ? ReadOffsetType.READ_FROM_STORE : ReadOffsetType.MEMORY_FIRST_THEN_STORE); + return this.offsetStore.readOffset(messageQueue, fromStore ? ReadOffsetType.READ_FROM_STORE : ReadOffsetType.MEMORY_FIRST_THEN_STORE); } - private long nextPullOffset(MessageQueue remoteQueue) { + public long committed(MessageQueue messageQueue) throws MQClientException { + checkServiceState(); + long offset = this.offsetStore.readOffset(messageQueue, ReadOffsetType.READ_FROM_STORE); + if (offset == -2) + throw new MQClientException("Fetch consume offset from broker exception", null); + return offset; + } + + private long nextPullOffset(MessageQueue messageQueue) { long offset = -1; - long seekOffset = assignedMessageQueue.getSeekOffset(remoteQueue); + long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); if (seekOffset != -1) { offset = seekOffset; - assignedMessageQueue.setSeekOffset(remoteQueue, -1); - assignedMessageQueue.updatePullOffset(remoteQueue, offset); + assignedMessageQueue.setSeekOffset(messageQueue, -1); + assignedMessageQueue.updatePullOffset(messageQueue, offset); } else { - offset = assignedMessageQueue.getPullOffset(remoteQueue); + offset = assignedMessageQueue.getPullOffset(messageQueue); if (offset == -1) { - offset = fetchConsumeOffset(remoteQueue, false); + offset = fetchConsumeOffset(messageQueue, false); if (offset == -1 && defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { offset = 0; } - assignedMessageQueue.updatePullOffset(remoteQueue, offset); - assignedMessageQueue.updateConsumeOffset(remoteQueue, offset); + assignedMessageQueue.updatePullOffset(messageQueue, offset); + assignedMessageQueue.updateConsumeOffset(messageQueue, offset); } } @@ -751,7 +762,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } long offset = nextPullOffset(messageQueue); - long pullDelayTimeMills = defaultLitePullConsumer.getPullDelayTimeMills(); + long pullDelayTimeMills = 0; try { SubscriptionData subscriptionData; @@ -775,9 +786,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { case OFFSET_ILLEGAL: log.warn("The pull request offset illegal, {}", pullResult.toString()); break; - case NO_NEW_MSG: - pullDelayTimeMills = PULL_TIME_DELAY_MILLS_WHEN_NO_NEW_MSG; - break; default: break; } diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index 68144c785e..8cceabcb2c 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -33,6 +33,7 @@ import org.apache.rocketmq.client.impl.FindBrokerResult; import org.apache.rocketmq.client.impl.MQAdminImpl; import org.apache.rocketmq.client.impl.MQClientAPIImpl; import org.apache.rocketmq.client.impl.MQClientManager; +import org.apache.rocketmq.client.impl.consumer.AssignedMessageQueue; import org.apache.rocketmq.client.impl.consumer.DefaultLitePullConsumerImpl; import org.apache.rocketmq.client.impl.consumer.PullAPIWrapper; import org.apache.rocketmq.client.impl.consumer.PullResultExt; @@ -44,7 +45,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; -import org.junit.After; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -56,6 +57,7 @@ import org.mockito.stubbing.Answer; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.failBecauseExceptionWasNotThrown; +import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyLong; @@ -77,7 +79,6 @@ public class DefaultLitePullConsumerTest { private RebalanceImpl rebalanceImpl; private OffsetStore offsetStore; - private DefaultLitePullConsumer litePullConsumer; private DefaultLitePullConsumerImpl litePullConsumerImpl; private String consumerGroup = "LitePullConsumerGroup"; private String topic = "LitePullConsumerTest"; @@ -85,20 +86,150 @@ public class DefaultLitePullConsumerTest { @Before public void init() throws Exception { - String groupName = consumerGroup + System.currentTimeMillis(); - litePullConsumer = new DefaultLitePullConsumer(groupName); - litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); - Field field = MQClientInstance.class.getDeclaredField("rebalanceService"); field.setAccessible(true); RebalanceService rebalanceService = (RebalanceService) field.get(mQClientFactory); field = RebalanceService.class.getDeclaredField("waitInterval"); field.setAccessible(true); field.set(rebalanceService, 100); + } - litePullConsumer.start(); + @Test + public void testAssign_PollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } - field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); + @Test + public void testSubscribe_PollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createSubscribeLitePullConsumer(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscribe_BroadcastPollMessageSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createBroadcastLitePullConsumer(); + try { + Set messageQueueSet = new HashSet(); + messageQueueSet.add(createMessageQueue()); + litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); + litePullConsumer.setPollTimeoutMillis(20 * 1000); + List result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSubscriptionType_AssignAndSubscribeExclusive() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.assign(Collections.singletonList(createMessageQueue())); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("Subscribe and assign are mutually exclusive."); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testFetchMesseageQueues_FetchMessageQueuesBeforeStart() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + litePullConsumer.fetchMessageQueues(topic); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("The consumer not running, please start it first."); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testSeek_SeekOffsetIllegal() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + try { + litePullConsumer.seek(messageQueue, -1); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("min offset = 0"); + } + + try { + litePullConsumer.seek(messageQueue, 1000); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("max offset = 100"); + } + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_SeekOffsetSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.seek(messageQueue, 50); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 50); + assertEquals(assignedMessageQueue.getConusmerOffset(messageQueue), 50); + litePullConsumer.shutdown(); + } + + @Test + public void testSeek_MessageQueueNotInAssignList() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + litePullConsumer.seek(createMessageQueue(), 0); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("The message queue is not in assigned list"); + } finally { + litePullConsumer.shutdown(); + } + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + + private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { + + Field field = DefaultLitePullConsumer.class.getDeclaredField("defaultLitePullConsumerImpl"); field.setAccessible(true); litePullConsumerImpl = (DefaultLitePullConsumerImpl) field.get(litePullConsumer); field = DefaultLitePullConsumerImpl.class.getDeclaredField("mQClientFactory"); @@ -156,96 +287,35 @@ public class DefaultLitePullConsumerTest { doReturn(123L).when(offsetStore).readOffset(any(MessageQueue.class), any(ReadOffsetType.class)); } - @After - public void terminate() { - litePullConsumer.shutdown(); - } - - @Test - public void testAssign_PollMessageSuccess() { - MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.setPullDelayTimeMills(60 * 1000); - litePullConsumer.assign(Collections.singletonList(messageQueue)); - List result = litePullConsumer.poll(); - assertThat(result.get(0).getTopic()).isEqualTo(topic); - assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); - } - - @Test - public void testSubscribe_PollMessageSuccess() throws MQClientException { - litePullConsumer.setPullDelayTimeMills(60 * 1000); + private DefaultLitePullConsumer createSubscribeLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); litePullConsumer.subscribe(topic, "*"); - Set messageQueueSet = new HashSet(); - messageQueueSet.add(createMessageQueue()); - litePullConsumerImpl.updateTopicSubscribeInfo(topic, messageQueueSet); - litePullConsumer.setPollTimeoutMillis(20 * 1000); - List result = litePullConsumer.poll(); - assertThat(result.get(0).getTopic()).isEqualTo(topic); - assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; } - @Test - public void testSubscriptionType_AssignAndSubscribeExclusive() throws MQClientException { - try { - litePullConsumer.subscribe(topic, "*"); - litePullConsumer.assign(Collections.singletonList(createMessageQueue())); - failBecauseExceptionWasNotThrown(IllegalStateException.class); - } catch (IllegalStateException e) { - assertThat(e).hasMessageContaining("Cannot select two subscription types at the same time."); - } + private DefaultLitePullConsumer createBroadcastLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.setMessageModel(MessageModel.BROADCASTING); + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; } - @Test - public void testFetchMesseageQueues_FetchMessageQueuesBeforeStart() throws MQClientException { - try { - DefaultLitePullConsumer litePullConsumer = createLitePullConsumer(); - litePullConsumer.fetchMessageQueues(topic); - failBecauseExceptionWasNotThrown(IllegalStateException.class); - } catch (IllegalStateException e) { - assertThat(e).hasMessageContaining("The consumer not running."); - } + private DefaultLitePullConsumer createStartLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; } - @Test - public void testSeek_SeekOffsetIllegal() throws MQClientException { - when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); - when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); - MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); - try { - litePullConsumer.seek(messageQueue, -1); - failBecauseExceptionWasNotThrown(MQClientException.class); - } catch (MQClientException e) { - assertThat(e).hasMessageContaining("min offset = 0"); - } - - try { - litePullConsumer.seek(messageQueue, 1000); - failBecauseExceptionWasNotThrown(MQClientException.class); - } catch (MQClientException e) { - assertThat(e).hasMessageContaining("max offset = 100"); - } - } - - @Test - public void testSeek_MessageQueueNotInAssignList() { - try { - litePullConsumer.seek(createMessageQueue(), 0); - failBecauseExceptionWasNotThrown(MQClientException.class); - } catch (MQClientException e) { - assertThat(e).hasMessageContaining("The message queue is not in assigned list"); - } - } - - private MessageQueue createMessageQueue() { - MessageQueue messageQueue = new MessageQueue(); - messageQueue.setBrokerName(brokerName); - messageQueue.setQueueId(0); - messageQueue.setTopic(topic); - return messageQueue; - } - - private DefaultLitePullConsumer createLitePullConsumer() { + private DefaultLitePullConsumer createNotStartLitePullConsumer() { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); return litePullConsumer; } @@ -258,4 +328,5 @@ public class DefaultLitePullConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } + } diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java similarity index 70% rename from example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java rename to example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java index b0a2b596ad..e638de1c96 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumer.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerAssign.java @@ -18,31 +18,35 @@ package org.apache.rocketmq.example.simple; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; -public class LitePullConsumer { +public class LitePullConsumerAssign { + + public static volatile boolean running = true; + public static void main(String[] args) throws Exception { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("please_rename_unique_group_name"); litePullConsumer.setAutoCommit(false); litePullConsumer.start(); Collection mqSet = litePullConsumer.fetchMessageQueues("TopicTest"); List list = new ArrayList<>(mqSet); - Collection assginMq = Collections.singletonList(list.get(0)); - litePullConsumer.assign(assginMq); - int size = 0; - litePullConsumer.seek(list.get(0), 26); - - while (true) { - List messageExts = litePullConsumer.poll(); - if (messageExts != null) { - size += messageExts.size(); + List assignList = new ArrayList<>(); + for (int i = 0; i < list.size() / 2; i++) { + assignList.add(list.get(i)); + } + litePullConsumer.assign(assignList); + litePullConsumer.seek(assignList.get(0), 10); + try { + while (running) { + List messageExts = litePullConsumer.poll(); + System.out.printf("%s %n", messageExts); + litePullConsumer.commitSync(); } - litePullConsumer.commitSync(); - System.out.printf("%s %d %n", messageExts, size); + } finally { + litePullConsumer.shutdown(); } } diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java new file mode 100644 index 0000000000..f0dc4d2adf --- /dev/null +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.example.simple; + +import java.util.List; +import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; +import org.apache.rocketmq.common.message.MessageExt; + +public class LitePullConsumerSubscribe { + + public static volatile boolean running = true; + + public static void main(String[] args) throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("please_rename_unique_group_name"); + litePullConsumer.subscribe("TopicTest", "*"); + litePullConsumer.start(); + try { + while (running) { + List messageExts = litePullConsumer.poll(); + if (messageExts != null) { + System.out.printf("%s%n", messageExts); + } + } + } finally { + litePullConsumer.shutdown(); + } + } +} From 4a4a464b25c91e215f627d87d90afd97970401e4 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Wed, 21 Aug 2019 09:23:59 +0800 Subject: [PATCH 11/14] Polish lite pull consumer and fix bug (#1402) 1. Fix the problem of seek function blocking. 2. Fix shutdown function bug. 3. Make the code cleaner 4. Add unit test. 5. Fix seek bug --- .../apache/rocketmq/client/Validators.java | 9 +- .../consumer/DefaultLitePullConsumer.java | 129 ++------- .../store/RemoteBrokerOffsetStore.java | 41 ++- .../impl/consumer/AssignedMessageQueue.java | 20 +- .../consumer/DefaultLitePullConsumerImpl.java | 266 +++++++++--------- .../client/impl/factory/MQClientInstance.java | 4 - .../consumer/DefaultLitePullConsumerTest.java | 22 +- .../simple/LitePullConsumerSubscribe.java | 4 +- 8 files changed, 191 insertions(+), 304 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/Validators.java b/client/src/main/java/org/apache/rocketmq/client/Validators.java index 5d6acc0f87..1b96cd058f 100644 --- a/client/src/main/java/org/apache/rocketmq/client/Validators.java +++ b/client/src/main/java/org/apache/rocketmq/client/Validators.java @@ -53,14 +53,17 @@ public class Validators { if (UtilAll.isBlank(group)) { throw new MQClientException("the specified group is blank", null); } + + if (group.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified group is longer than group max length 255.", null); + } + if (!regularExpressionMatcher(group, PATTERN)) { throw new MQClientException(String.format( "the specified group[%s] contains illegal characters, allowing only %s", group, VALID_PATTERN_STR), null); } - if (group.length() > CHARACTER_MAX_LENGTH) { - throw new MQClientException("the specified group is longer than group max length 255.", null); - } + } /** diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 33d9e5fc9f..5a2189f327 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -128,6 +128,11 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon */ private long pollTimeoutMillis = 1000 * 5; + /** + * Interval time in in milliseconds for checking changes in topic metadata. + */ + private long topicMetadataCheckIntervalMillis = 30 * 1000; + /** * Default constructor. */ @@ -175,202 +180,92 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon defaultLitePullConsumerImpl = new DefaultLitePullConsumerImpl(this, rpcHook); } - /** - * Start the consumer - */ @Override public void start() throws MQClientException { this.defaultLitePullConsumerImpl.start(); } - /** - * Shutdown the consumer - */ @Override public void shutdown() { this.defaultLitePullConsumerImpl.shutdown(); } - /** - * Subscribe some topic with subExpression - * - * @param subExpression subscription expression.it only support or operation such as "tag1 || tag2 || tag3"
if - * null or * expression,meaning subscribe all - * @throws MQClientException if there is any client error. - */ @Override public void subscribe(String topic, String subExpression) throws MQClientException { this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), subExpression); } - /** - * Subscribe some topic with selector. - * - * @param messageSelector message selector({@link MessageSelector}), can be null. - * @throws MQClientException if there is any client error. - */ @Override public void subscribe(String topic, MessageSelector messageSelector) throws MQClientException { this.defaultLitePullConsumerImpl.subscribe(withNamespace(topic), messageSelector); } - /** - * Unsubscribe consumption some topic - * - * @param topic Message topic that needs to be unsubscribe. - */ @Override public void unsubscribe(String topic) { this.defaultLitePullConsumerImpl.unsubscribe(withNamespace(topic)); } - /** - * Manually assign a list of message queues to this consumer. This interface does not allow for incremental - * assignment and will replace the previous assignment (if there is one). - * - * @param messageQueues Message queues that needs to be assigned. - */ @Override public void assign(Collection messageQueues) { defaultLitePullConsumerImpl.assign(queuesWithNamespace(messageQueues)); } - /** - * Fetch data for the topics or partitions specified using assign API - * - * @return list of message, can be null. - */ @Override public List poll() { return defaultLitePullConsumerImpl.poll(this.getPollTimeoutMillis()); } - /** - * Fetch data for the topics or partitions specified using assign API - * - * @param timeout The amount time, in milliseconds, spent waiting in poll if data is not available. Must not be - * negative - * @return list of message, can be null. - */ @Override public List poll(long timeout) { return defaultLitePullConsumerImpl.poll(timeout); } - /** - * Overrides the fetch offsets that the consumer will use on the next poll. If this API is invoked for the same - * message queue more than once, the latest offset will be used on the next poll(). Note that you may lose data if - * this API is arbitrarily used in the middle of consumption. - * - * @param messageQueue - * @param offset - */ @Override public void seek(MessageQueue messageQueue, long offset) throws MQClientException { this.defaultLitePullConsumerImpl.seek(queueWithNamespace(messageQueue), offset); } - /** - * Suspend pulling from the requested message queues. - * - * Because of the implementation of pre-pull, fetch data in {@link #poll()} will not stop immediately until the - * messages of the requested message queues drain. - * - * Note that this method does not affect message queue subscription. In particular, it does not cause a group - * rebalance. - * - * @param messageQueues Message queues that needs to be paused. - */ @Override public void pause(Collection messageQueues) { this.defaultLitePullConsumerImpl.pause(queuesWithNamespace(messageQueues)); } - /** - * Resume specified message queues which have been paused with {@link #pause(Collection)}. - * - * @param messageQueues Message queues that needs to be resumed. - */ @Override public void resume(Collection messageQueues) { this.defaultLitePullConsumerImpl.resume(queuesWithNamespace(messageQueues)); } - /** - * Get metadata about the message queues for a given topic. - * - * @param topic The topic that need to get metadata. - * @return collection of message queues - * @throws MQClientException if there is any client error. - */ @Override public Collection fetchMessageQueues(String topic) throws MQClientException { return this.defaultLitePullConsumerImpl.fetchMessageQueues(withNamespace(topic)); } - /** - * Look up the offsets for the given message queue by timestamp. The returned offset for each message queue is the - * earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message - * queue. - * - * @param messageQueue Message queues that needs to get offset by timestamp. - * @param timestamp - * @return offset - * @throws MQClientException if there is any client error. - */ @Override public Long offsetForTimestamp(MessageQueue messageQueue, Long timestamp) throws MQClientException { return this.defaultLitePullConsumerImpl.searchOffset(queueWithNamespace(messageQueue), timestamp); } - /** - * Register a callback for sensing topic metadata changes. - * - * @param topic The topic that need to monitor. - * @param topicMessageQueueChangeListener Callback when topic metadata changes. - * @throws MQClientException if there is any client error. - */ @Override public void registerTopicMessageQueueChangeListener(String topic, TopicMessageQueueChangeListener topicMessageQueueChangeListener) throws MQClientException { this.defaultLitePullConsumerImpl.registerTopicMessageQueueChangeListener(withNamespace(topic), topicMessageQueueChangeListener); } - /** - * Manually commit consume offset. - */ @Override public void commitSync() { this.defaultLitePullConsumerImpl.commitSync(); } - /** - * Get the last committed offset for the given message queue. - * - * @param messageQueue - * @return offset, if offset equals -1 means no offset in broker. - * @throws MQClientException if there is any client error. - */ @Override public Long committed(MessageQueue messageQueue) throws MQClientException { return this.defaultLitePullConsumerImpl.committed(messageQueue); } - /** - * Whether to enable auto-commit consume offset. - * - * @return true if enable auto-commit, false if disable auto-commit. - */ @Override public boolean isAutoCommit() { return autoCommit; } - /** - * Set whether to enable auto-commit consume offset. - * - * @param autoCommit Whether to enable auto-commit. - */ @Override public void setAutoCommit(boolean autoCommit) { this.autoCommit = autoCommit; @@ -392,12 +287,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon this.autoCommitIntervalMillis = autoCommitIntervalMillis; } - public int getPullBatchNums() { + public int getPullBatchSize() { return pullBatchSize; } - public void setPullBatchNums(int pullBatchNums) { - this.pullBatchSize = pullBatchNums; + public void setPullBatchSize(int pullBatchSize) { + this.pullBatchSize = pullBatchSize; } public long getPullThresholdForAll() { @@ -503,4 +398,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; } + + public long getTopicMetadataCheckIntervalMillis() { + return topicMetadataCheckIntervalMillis; + } + + public void setTopicMetadataCheckIntervalMillis(long topicMetadataCheckIntervalMillis) { + this.topicMetadataCheckIntervalMillis = topicMetadataCheckIntervalMillis; + } } diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java index 63dc525e7e..6b76238371 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -117,25 +117,24 @@ public class RemoteBrokerOffsetStore implements OffsetStore { return; final HashSet unusedMQ = new HashSet(); - if (!mqs.isEmpty()) { - for (Map.Entry entry : this.offsetTable.entrySet()) { - MessageQueue mq = entry.getKey(); - AtomicLong offset = entry.getValue(); - if (offset != null) { - if (mqs.contains(mq)) { - try { - this.updateConsumeOffsetToBroker(mq, offset.get()); - log.info("[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", - this.groupName, - this.mQClientFactory.getClientId(), - mq, - offset.get()); - } catch (Exception e) { - log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); - } - } else { - unusedMQ.add(mq); + + for (Map.Entry entry : this.offsetTable.entrySet()) { + MessageQueue mq = entry.getKey(); + AtomicLong offset = entry.getValue(); + if (offset != null) { + if (mqs.contains(mq)) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + log.info("[persistAll] Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", + this.groupName, + this.mQClientFactory.getClientId(), + mq, + offset.get()); + } catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); } + } else { + unusedMQ.add(mq); } } } @@ -187,8 +186,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore { } /** - * Update the Consumer Offset in one way, once the Master is off, updated to Slave, - * here need to be optimized. + * Update the Consumer Offset in one way, once the Master is off, updated to Slave, here need to be optimized. */ private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { @@ -196,8 +194,7 @@ public class RemoteBrokerOffsetStore implements OffsetStore { } /** - * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, - * here need to be optimized. + * Update the Consumer Offset synchronously, once the Master is off, updated to Slave, here need to be optimized. */ @Override public void updateConsumeOffsetToBroker(MessageQueue mq, long offset, boolean isOneway) throws RemotingException, diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java index 609fc4d1ef..c0c6f6030c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/AssignedMessageQueue.java @@ -21,12 +21,11 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.message.MessageQueue; public class AssignedMessageQueue { - private ConcurrentHashMap assignedMessageQueueState; + private final ConcurrentHashMap assignedMessageQueueState; private RebalanceImpl rebalanceImpl; @@ -54,7 +53,6 @@ public class AssignedMessageQueue { for (MessageQueue messageQueue : messageQueues) { MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { - messageQueueState.getPausedLatch().reset(); messageQueueState.setPaused(true); } } @@ -65,7 +63,6 @@ public class AssignedMessageQueue { MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); if (assignedMessageQueueState.get(messageQueue) != null) { messageQueueState.setPaused(false); - messageQueueState.getPausedLatch().reset(); } } } @@ -123,14 +120,6 @@ public class AssignedMessageQueue { return -1; } - public CountDownLatch2 getPausedLatch(MessageQueue messageQueue) { - MessageQueueState messageQueueState = assignedMessageQueueState.get(messageQueue); - if (messageQueueState != null) { - return messageQueueState.getPausedLatch(); - } - return null; - } - public void updateAssignedMessageQueue(String topic, Collection assigned) { synchronized (this.assignedMessageQueueState) { Iterator> it = this.assignedMessageQueueState.entrySet().iterator(); @@ -195,9 +184,8 @@ public class AssignedMessageQueue { private volatile long pullOffset = -1; private volatile long consumeOffset = -1; private volatile long seekOffset = -1; - private CountDownLatch2 pausedLatch = new CountDownLatch2(1); - public MessageQueueState(MessageQueue messageQueue, ProcessQueue processQueue) { + private MessageQueueState(MessageQueue messageQueue, ProcessQueue processQueue) { this.messageQueue = messageQueue; this.processQueue = processQueue; } @@ -249,9 +237,5 @@ public class AssignedMessageQueue { public void setSeekOffset(long seekOffset) { this.seekOffset = seekOffset; } - - public CountDownLatch2 getPausedLatch() { - return pausedLatch; - } } } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 2fb33385d5..5dd9c352dc 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -53,7 +53,6 @@ import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; -import org.apache.rocketmq.common.CountDownLatch2; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.ThreadFactoryImpl; @@ -142,6 +141,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { private long nextAutoCommitDeadline = -1L; + private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + public DefaultLitePullConsumerImpl(final DefaultLitePullConsumer defaultLitePullConsumer, final RPCHook rpcHook) { this.defaultLitePullConsumer = defaultLitePullConsumer; this.rpcHook = rpcHook; @@ -206,8 +207,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private int nextPullBatchNums() { - return Math.min(this.defaultLitePullConsumer.getPullBatchNums(), consumeRequestCache.remainingCapacity()); + private int nextPullBatchSize() { + return Math.min(this.defaultLitePullConsumer.getPullBatchSize(), consumeRequestCache.remainingCapacity()); } public synchronized void shutdown() { @@ -217,13 +218,11 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { case RUNNING: persistConsumerOffset(); this.mQClientFactory.unregisterConsumer(this.defaultLitePullConsumer.getConsumerGroup()); - this.mQClientFactory.shutdown(); - log.info("the consumer [{}] shutdown OK", this.defaultLitePullConsumer.getConsumerGroup()); scheduledThreadPoolExecutor.shutdown(); scheduledExecutorService.shutdown(); + this.mQClientFactory.shutdown(); this.serviceState = ServiceState.SHUTDOWN_ALREADY; - break; - case SHUTDOWN_ALREADY: + log.info("the consumer [{}] shutdown OK", this.defaultLitePullConsumer.getConsumerGroup()); break; default: break; @@ -237,79 +236,28 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { this.checkConfig(); - this.copySubscription(); - if (this.defaultLitePullConsumer.getMessageModel() == MessageModel.CLUSTERING) { this.defaultLitePullConsumer.changeInstanceNameToPID(); } - this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); + initMQClientFactory(); - this.rebalanceImpl.setConsumerGroup(this.defaultLitePullConsumer.getConsumerGroup()); - this.rebalanceImpl.setMessageModel(this.defaultLitePullConsumer.getMessageModel()); - this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()); - this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + initRebalanceImpl(); - this.pullAPIWrapper = new PullAPIWrapper( - mQClientFactory, - this.defaultLitePullConsumer.getConsumerGroup(), isUnitMode()); - this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + initPullAPIWrapper(); - if (this.defaultLitePullConsumer.getOffsetStore() != null) { - this.offsetStore = this.defaultLitePullConsumer.getOffsetStore(); - } else { - switch (this.defaultLitePullConsumer.getMessageModel()) { - case BROADCASTING: - this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); - break; - case CLUSTERING: - this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); - break; - default: - break; - } - this.defaultLitePullConsumer.setOffsetStore(this.offsetStore); - } - - this.offsetStore.load(); - - boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); - if (!registerOK) { - this.serviceState = ServiceState.CREATE_JUST; - - throw new MQClientException("The consumer group[" + this.defaultLitePullConsumer.getConsumerGroup() - + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), - null); - } + initOffsetStore(); mQClientFactory.start(); - if (subscriptionType == SubscriptionType.SUBSCRIBE) { - updateTopicSubscribeInfoWhenSubscriptionChanged(); - } - if (subscriptionType == SubscriptionType.ASSIGN) { - updateAssignPullTask(assignedMessageQueue.messageQueues()); - } + startScheduleTask(); - scheduledExecutorService.scheduleAtFixedRate( - new Runnable() { - @Override - public void run() { - try { - fetchTopicMessageQueuesAndCompare(); - } catch (Exception e) { - log.error("ScheduledTask fetchMessageQueuesAndCompare exception", e); - } - } - }, 1000 * 20, 1000 * 30, TimeUnit.MILLISECONDS); + this.serviceState = ServiceState.RUNNING; log.info("the consumer [{}] start OK", this.defaultLitePullConsumer.getConsumerGroup()); - this.serviceState = ServiceState.RUNNING; - for (String topic : topicMessageQueueChangeListenerMap.keySet()) { - Set messageQueues = fetchMessageQueues(topic); - messageQueuesForTopic.put(topic, messageQueues); - } - this.mQClientFactory.checkClientInBroker(); + + operateAfterRunning(); + break; case RUNNING: case START_FAILED: @@ -323,19 +271,87 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - private void checkConfig() throws MQClientException { - // check consumerGroup - Validators.checkGroup(this.defaultLitePullConsumer.getConsumerGroup()); + private void initMQClientFactory() throws MQClientException { + this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultLitePullConsumer, this.rpcHook); + boolean registerOK = mQClientFactory.registerConsumer(this.defaultLitePullConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; - // consumerGroup - if (null == this.defaultLitePullConsumer.getConsumerGroup()) { - throw new MQClientException( - "consumerGroup is null" - + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), + throw new MQClientException("The consumer group[" + this.defaultLitePullConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); } + } - // consumerGroup + private void initRebalanceImpl() { + this.rebalanceImpl.setConsumerGroup(this.defaultLitePullConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultLitePullConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + } + + private void initPullAPIWrapper() { + this.pullAPIWrapper = new PullAPIWrapper( + mQClientFactory, + this.defaultLitePullConsumer.getConsumerGroup(), isUnitMode()); + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + } + + private void initOffsetStore() throws MQClientException { + if (this.defaultLitePullConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultLitePullConsumer.getOffsetStore(); + } else { + switch (this.defaultLitePullConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultLitePullConsumer.getConsumerGroup()); + break; + default: + break; + } + this.defaultLitePullConsumer.setOffsetStore(this.offsetStore); + } + this.offsetStore.load(); + } + + private void startScheduleTask() { + scheduledExecutorService.scheduleAtFixedRate( + new Runnable() { + @Override + public void run() { + try { + fetchTopicMessageQueuesAndCompare(); + } catch (Exception e) { + log.error("ScheduledTask fetchMessageQueuesAndCompare exception", e); + } + } + }, 1000 * 20, this.getDefaultLitePullConsumer().getTopicMetadataCheckIntervalMillis(), TimeUnit.MILLISECONDS); + } + + private void operateAfterRunning() throws MQClientException { + // If subscribe function invoke before start function, then update topic subscribe info after initialization. + if (subscriptionType == SubscriptionType.SUBSCRIBE) { + updateTopicSubscribeInfoWhenSubscriptionChanged(); + } + // If assign function invoke before start function, then update pull task after initialization. + if (subscriptionType == SubscriptionType.ASSIGN) { + updateAssignPullTask(assignedMessageQueue.messageQueues()); + } + + for (String topic : topicMessageQueueChangeListenerMap.keySet()) { + Set messageQueues = fetchMessageQueues(topic); + messageQueuesForTopic.put(topic, messageQueues); + } + this.mQClientFactory.checkClientInBroker(); + } + + private void checkConfig() throws MQClientException { + // Check consumerGroup + Validators.checkGroup(this.defaultLitePullConsumer.getConsumerGroup()); + + // Check consumerGroup name is not equal default consumer group name. if (this.defaultLitePullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { throw new MQClientException( "consumerGroup can not equal " @@ -345,7 +361,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null); } - // messageModel + // Check messageModel is not null. if (null == this.defaultLitePullConsumer.getMessageModel()) { throw new MQClientException( "messageModel is null" @@ -353,7 +369,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null); } - // allocateMessageQueueStrategy + // Check allocateMessageQueueStrategy is not null if (null == this.defaultLitePullConsumer.getAllocateMessageQueueStrategy()) { throw new MQClientException( "allocateMessageQueueStrategy is null" @@ -361,7 +377,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { null); } - // allocateMessageQueueStrategy if (this.defaultLitePullConsumer.getConsumerTimeoutMillisWhenSuspend() < this.defaultLitePullConsumer.getBrokerSuspendMaxTimeMillis()) { throw new MQClientException( "Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis" @@ -374,24 +389,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return pullAPIWrapper; } - private void copySubscription() throws MQClientException { - try { - switch (this.defaultLitePullConsumer.getMessageModel()) { - case BROADCASTING: - break; - case CLUSTERING: - /* - * Retry topic will be support in the future. - */ - break; - default: - break; - } - } catch (Exception e) { - throw new MQClientException("subscription exception", e); - } - } - private void startPullTask(Collection mqSet) { for (MessageQueue messageQueue : mqSet) { if (!this.taskTable.containsKey(messageQueue)) { @@ -526,7 +523,8 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } catch (InterruptedException ignore) { } - return null; + + return Collections.emptyList(); } public void pause(Collection messageQueues) { @@ -547,29 +545,13 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } long minOffset = minOffset(messageQueue); long maxOffset = maxOffset(messageQueue); - if (offset < minOffset || offset > maxOffset) + if (offset < minOffset || offset > maxOffset) { throw new MQClientException("Seek offset illegal, seek offset = " + offset + ", min offset = " + minOffset + ", max offset = " + maxOffset, null); - try { - assignedMessageQueue.pause(Collections.singletonList(messageQueue)); - CountDownLatch2 pausedLatch = assignedMessageQueue.getPausedLatch(messageQueue); - if (pausedLatch != null) { - pausedLatch.await(2, TimeUnit.SECONDS); - } - ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); - if (processQueue != null) { - processQueue.clear(); - } - Iterator iter = consumeRequestCache.iterator(); - while (iter.hasNext()) { - if (iter.next().getMessageQueue().equals(messageQueue)) - iter.remove(); - } + } + final Object objLock = messageQueueLock.fetchLockObject(messageQueue); + synchronized (objLock) { assignedMessageQueue.setSeekOffset(messageQueue, offset); - assignedMessageQueue.updateConsumeOffset(messageQueue, offset); - } catch (Exception e) { - log.error("Seek offset failed.", e); - } finally { - assignedMessageQueue.resume(Collections.singletonList(messageQueue)); + clearMessageQueueInCache(messageQueue); } } @@ -611,8 +593,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } } - if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) + if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { offsetStore.persistAll(assignedMessageQueue.messageQueues()); + } } catch (Exception e) { log.error("An error occurred when update consume offset synchronously.", e); } @@ -631,8 +614,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } } - if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) + if (defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { offsetStore.persistAll(assignedMessageQueue.messageQueues()); + } } catch (Exception e) { log.error("An error occurred when update consume offset Automatically."); } @@ -665,13 +649,25 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return offset; } + private void clearMessageQueueInCache(MessageQueue messageQueue) { + ProcessQueue processQueue = assignedMessageQueue.getProcessQueue(messageQueue); + if (processQueue != null) { + processQueue.clear(); + } + Iterator iter = consumeRequestCache.iterator(); + while (iter.hasNext()) { + if (iter.next().getMessageQueue().equals(messageQueue)) + iter.remove(); + } + } + private long nextPullOffset(MessageQueue messageQueue) { long offset = -1; long seekOffset = assignedMessageQueue.getSeekOffset(messageQueue); if (seekOffset != -1) { offset = seekOffset; + assignedMessageQueue.updateConsumeOffset(messageQueue, offset); assignedMessageQueue.setSeekOffset(messageQueue, -1); - assignedMessageQueue.updatePullOffset(messageQueue, offset); } else { offset = assignedMessageQueue.getPullOffset(messageQueue); if (offset == -1) { @@ -679,11 +675,9 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { if (offset == -1 && defaultLitePullConsumer.getMessageModel() == MessageModel.BROADCASTING) { offset = 0; } - assignedMessageQueue.updatePullOffset(messageQueue, offset); - assignedMessageQueue.updateConsumeOffset(messageQueue, offset); } + assignedMessageQueue.updateConsumeOffset(messageQueue, offset); } - return offset; } @@ -706,9 +700,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { if (!this.isCancelled()) { if (assignedMessageQueue.isPaused(messageQueue)) { - CountDownLatch2 pasuedLatch = assignedMessageQueue.getPausedLatch(messageQueue); - if (pasuedLatch != null) - pasuedLatch.countDown(); scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_PAUSE, TimeUnit.MILLISECONDS); log.debug("Message Queue: {} has been paused!", messageQueue); return; @@ -721,7 +712,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return; } - if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchNums() > defaultLitePullConsumer.getPullThresholdForAll()) { + if (consumeRequestCache.size() * defaultLitePullConsumer.getPullBatchSize() > defaultLitePullConsumer.getPullThresholdForAll()) { scheduledThreadPoolExecutor.schedule(this, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL, TimeUnit.MILLISECONDS); if ((consumeRequestFlowControlTimes++ % 1000) == 0) log.warn("The consume request count exceeds threshold {}, so do flow control, consume request count={}, flowControlTimes={}", consumeRequestCache.size(), consumeRequestFlowControlTimes); @@ -764,7 +755,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { long offset = nextPullOffset(messageQueue); long pullDelayTimeMills = 0; try { - SubscriptionData subscriptionData; if (subscriptionType == SubscriptionType.SUBSCRIBE) { String topic = this.messageQueue.getTopic(); @@ -775,12 +765,15 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { topic, SubscriptionData.SUB_ALL); } - PullResult pullResult = pull(messageQueue, subscriptionData, offset, nextPullBatchNums()); + PullResult pullResult = pull(messageQueue, subscriptionData, offset, nextPullBatchSize()); switch (pullResult.getPullStatus()) { case FOUND: - if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty()) { - processQueue.putMessage(pullResult.getMsgFoundList()); - submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + final Object objLock = messageQueueLock.fetchLockObject(messageQueue); + synchronized (objLock) { + if (pullResult.getMsgFoundList() != null && !pullResult.getMsgFoundList().isEmpty() && assignedMessageQueue.getSeekOffset(messageQueue) == -1) { + processQueue.putMessage(pullResult.getMsgFoundList()); + submitConsumeRequest(new ConsumeRequest(pullResult.getMsgFoundList(), messageQueue, processQueue)); + } } break; case OFFSET_ILLEGAL: @@ -982,11 +975,6 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { return offsetStore; } - public void registerFilterMessageHook(final FilterMessageHook hook) { - this.filterMessageHookList.add(hook); - log.info("register FilterMessageHook Hook, {}", hook.hookName()); - } - public DefaultLitePullConsumer getDefaultLitePullConsumer() { return defaultLitePullConsumer; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java index 2a4fb7dfa6..4f9d42cc57 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java @@ -246,10 +246,6 @@ public class MQClientInstance { log.info("the client factory [{}] start OK", this.clientId); this.serviceState = ServiceState.RUNNING; break; - case RUNNING: - break; - case SHUTDOWN_ALREADY: - break; case START_FAILED: throw new MQClientException("The Factory object[" + this.getClientId() + "] has been created before, and failed.", null); default: diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index 8cceabcb2c..cbcc739266 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -202,7 +202,6 @@ public class DefaultLitePullConsumerTest { field.setAccessible(true); AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 50); - assertEquals(assignedMessageQueue.getConusmerOffset(messageQueue), 50); litePullConsumer.shutdown(); } @@ -219,6 +218,26 @@ public class DefaultLitePullConsumerTest { } } + @Test + public void testPauseAndResume_Success() throws Exception { + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.pause(Collections.singletonList(messageQueue)); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + List result = litePullConsumer.poll(); + assertThat(result.isEmpty()).isTrue(); + litePullConsumer.resume(Collections.singletonList(messageQueue)); + result = litePullConsumer.poll(); + assertThat(result.get(0).getTopic()).isEqualTo(topic); + assertThat(result.get(0).getBody()).isEqualTo(new byte[] {'a'}); + } finally { + litePullConsumer.shutdown(); + } + } + private MessageQueue createMessageQueue() { MessageQueue messageQueue = new MessageQueue(); messageQueue.setBrokerName(brokerName); @@ -328,5 +347,4 @@ public class DefaultLitePullConsumerTest { } return new PullResultExt(pullStatus, requestHeader.getQueueOffset() + messageExtList.size(), 123, 2048, messageExtList, 0, outputStream.toByteArray()); } - } diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java index f0dc4d2adf..1bfe49d736 100644 --- a/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java +++ b/example/src/main/java/org/apache/rocketmq/example/simple/LitePullConsumerSubscribe.java @@ -31,9 +31,7 @@ public class LitePullConsumerSubscribe { try { while (running) { List messageExts = litePullConsumer.poll(); - if (messageExts != null) { - System.out.printf("%s%n", messageExts); - } + System.out.printf("%s%n", messageExts); } } finally { litePullConsumer.shutdown(); From 2126929a12840420270bb771b2ac8160d65e722b Mon Sep 17 00:00:00 2001 From: vongosling Date: Thu, 22 Aug 2019 14:08:31 +0800 Subject: [PATCH 12/14] Deprecated related core pull consumer classes --- .../rocketmq/client/consumer/DefaultMQPullConsumer.java | 2 +- .../client/impl/consumer/DefaultMQPullConsumerImpl.java | 5 +++++ .../apache/rocketmq/client/impl/consumer/RebalanceImpl.java | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java index e6d3f70484..0876a94e4c 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -36,7 +36,7 @@ import org.apache.rocketmq.remoting.exception.RemotingException; /** * Default pulling consumer. - * This Consumer will be removed in 2022, and a better implementation {@link DefaultLitePullConsumer} is recommend to use + * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumer} is recommend to use * in the scenario of actively pulling messages. */ @Deprecated diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java index 3c98385e13..ed9d690891 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -66,6 +66,11 @@ import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; +/** + * This class will be removed in 2022, and a better implementation {@link DefaultLitePullConsumerImpl} is recommend to use + * in the scenario of actively pulling messages. + */ +@Deprecated public class DefaultMQPullConsumerImpl implements MQConsumerInner { private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPullConsumer defaultMQPullConsumer; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java index 9ad07c7e44..146fce6e1e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -41,8 +41,10 @@ import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; /** - * Base class for rebalance algorithm + * This class will be removed in 2022, and a better implementation {@link RebalanceLitePullImpl} is recommend to use + * in the scenario of actively pulling messages. */ +@Deprecated public abstract class RebalanceImpl { protected static final InternalLogger log = ClientLogger.getLog(); protected final ConcurrentMap processQueueTable = new ConcurrentHashMap(64); From 2ab9e850fa69de02b62c9da7d64a109f8b7d74a9 Mon Sep 17 00:00:00 2001 From: vongosling Date: Thu, 22 Aug 2019 14:19:14 +0800 Subject: [PATCH 13/14] Polish the consumer group doc --- .../rocketmq/client/consumer/DefaultLitePullConsumer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java index 5a2189f327..99976d55d3 100644 --- a/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java +++ b/client/src/main/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumer.java @@ -35,7 +35,12 @@ public class DefaultLitePullConsumer extends ClientConfig implements LitePullCon private final DefaultLitePullConsumerImpl defaultLitePullConsumerImpl; /** - * Do the same thing for the same Group, the application must be set,and guarantee Globally unique + * Consumers belonging to the same consumer group share a group id. The consumers in a group then + * divides the topic as fairly amongst themselves as possible by establishing that each queue is only + * consumed by a single consumer from the group. If all consumers are from the same group, it functions + * as a traditional message queue. Each message would be consumed by one consumer of the group only. + * When multiple consumer groups exist, the flow of the data consumption model aligns with the traditional + * publish-subscribe model. The messages are broadcast to all consumer groups. */ private String consumerGroup; From 67751d818df898b642c48b8e7500d1e9c4926719 Mon Sep 17 00:00:00 2001 From: King <794220751@qq.com> Date: Thu, 22 Aug 2019 15:26:12 +0800 Subject: [PATCH 14/14] Add unit test for lite pull consumer (#1410) * Add unit test for lite pull consumer. * Add synchronized to poll function. --- .../consumer/DefaultLitePullConsumerImpl.java | 4 +- .../consumer/DefaultLitePullConsumerTest.java | 212 +++++++++++++++--- 2 files changed, 181 insertions(+), 35 deletions(-) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java index 5dd9c352dc..5217a31539 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/DefaultLitePullConsumerImpl.java @@ -327,7 +327,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { log.error("ScheduledTask fetchMessageQueuesAndCompare exception", e); } } - }, 1000 * 20, this.getDefaultLitePullConsumer().getTopicMetadataCheckIntervalMillis(), TimeUnit.MILLISECONDS); + }, 1000 * 10, this.getDefaultLitePullConsumer().getTopicMetadataCheckIntervalMillis(), TimeUnit.MILLISECONDS); } private void operateAfterRunning() throws MQClientException { @@ -491,7 +491,7 @@ public class DefaultLitePullConsumerImpl implements MQConsumerInner { } } - public List poll(long timeout) { + public synchronized List poll(long timeout) { try { checkServiceState(); if (timeout < 0) diff --git a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java index cbcc739266..7d496acf02 100644 --- a/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/consumer/DefaultLitePullConsumerTest.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.client.impl.consumer.PullResultExt; import org.apache.rocketmq.client.impl.consumer.RebalanceImpl; import org.apache.rocketmq.client.impl.consumer.RebalanceService; import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.message.MessageClientExt; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; @@ -83,6 +84,7 @@ public class DefaultLitePullConsumerTest { private String consumerGroup = "LitePullConsumerGroup"; private String topic = "LitePullConsumerTest"; private String brokerName = "BrokerA"; + private boolean flag = false; @Before public void init() throws Exception { @@ -155,7 +157,7 @@ public class DefaultLitePullConsumerTest { } @Test - public void testFetchMesseageQueues_FetchMessageQueuesBeforeStart() throws Exception { + public void testFetchMessageQueues_FetchMessageQueuesBeforeStart() throws Exception { DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); try { litePullConsumer.fetchMessageQueues(topic); @@ -167,6 +169,22 @@ public class DefaultLitePullConsumerTest { } } + @Test + public void testSeek_SeekOffsetSuccess() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); + when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(500L); + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + long offset = litePullConsumer.committed(messageQueue); + litePullConsumer.seek(messageQueue, offset); + Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); + field.setAccessible(true); + AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); + assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), offset); + litePullConsumer.shutdown(); + } + @Test public void testSeek_SeekOffsetIllegal() throws Exception { DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); @@ -190,21 +208,6 @@ public class DefaultLitePullConsumerTest { litePullConsumer.shutdown(); } - @Test - public void testSeek_SeekOffsetSuccess() throws Exception { - DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); - when(mQAdminImpl.minOffset(any(MessageQueue.class))).thenReturn(0L); - when(mQAdminImpl.maxOffset(any(MessageQueue.class))).thenReturn(100L); - MessageQueue messageQueue = createMessageQueue(); - litePullConsumer.assign(Collections.singletonList(messageQueue)); - litePullConsumer.seek(messageQueue, 50); - Field field = DefaultLitePullConsumerImpl.class.getDeclaredField("assignedMessageQueue"); - field.setAccessible(true); - AssignedMessageQueue assignedMessageQueue = (AssignedMessageQueue) field.get(litePullConsumerImpl); - assertEquals(assignedMessageQueue.getSeekOffset(messageQueue), 50); - litePullConsumer.shutdown(); - } - @Test public void testSeek_MessageQueueNotInAssignList() throws Exception { DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); @@ -216,6 +219,34 @@ public class DefaultLitePullConsumerTest { } finally { litePullConsumer.shutdown(); } + + litePullConsumer = createSubscribeLitePullConsumer(); + try { + litePullConsumer.seek(createMessageQueue(), 0); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("The message queue is not in assigned list, may be rebalancing"); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testOffsetForTimestamp_FailedAndSuccess() throws Exception { + MessageQueue messageQueue = createMessageQueue(); + DefaultLitePullConsumer litePullConsumer = createNotStartLitePullConsumer(); + try { + litePullConsumer.offsetForTimestamp(messageQueue, 123456L); + failBecauseExceptionWasNotThrown(IllegalStateException.class); + } catch (IllegalStateException e) { + assertThat(e).hasMessageContaining("The consumer not running, please start it first."); + } finally { + litePullConsumer.shutdown(); + } + doReturn(123L).when(mQAdminImpl).searchOffset(any(MessageQueue.class), anyLong()); + litePullConsumer = createStartLitePullConsumer(); + long offset = litePullConsumer.offsetForTimestamp(messageQueue, 123456L); + assertThat(offset).isEqualTo(123L); } @Test @@ -238,12 +269,120 @@ public class DefaultLitePullConsumerTest { } } - private MessageQueue createMessageQueue() { - MessageQueue messageQueue = new MessageQueue(); - messageQueue.setBrokerName(brokerName); - messageQueue.setQueueId(0); - messageQueue.setTopic(topic); - return messageQueue; + @Test + public void testRegisterTopicMessageQueueChangeListener_Success() throws Exception { + flag = false; + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + doReturn(Collections.emptySet()).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString()); + litePullConsumer.setTopicMetadataCheckIntervalMillis(10); + litePullConsumer.registerTopicMessageQueueChangeListener(topic, new TopicMessageQueueChangeListener() { + @Override public void onChanged(String topic, Set messageQueues) { + flag = true; + } + }); + Set set = new HashSet(); + set.add(createMessageQueue()); + doReturn(set).when(mQAdminImpl).fetchSubscribeMessageQueues(anyString()); + Thread.sleep(11 * 1000); + assertThat(flag).isTrue(); + } + + @Test + public void testFlowControl_Success() throws Exception { + DefaultLitePullConsumer litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdForAll(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdForQueue(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setPullThresholdSizeForQueue(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = createStartLitePullConsumer(); + try { + MessageQueue messageQueue = createMessageQueue(); + litePullConsumer.setConsumeMaxSpan(-1); + litePullConsumer.assign(Collections.singletonList(messageQueue)); + litePullConsumer.setPollTimeoutMillis(500); + List result = litePullConsumer.poll(); + assertThat(result).isEmpty(); + } finally { + litePullConsumer.shutdown(); + } + } + + @Test + public void testCheckConfig_Exception() { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(MixAll.DEFAULT_CONSUMER_GROUP); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("consumerGroup can not equal"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setMessageModel(null); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("messageModel is null"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setAllocateMessageQueueStrategy(null); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("allocateMessageQueueStrategy is null"); + } finally { + litePullConsumer.shutdown(); + } + + litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setConsumerTimeoutMillisWhenSuspend(1); + try { + litePullConsumer.start(); + failBecauseExceptionWasNotThrown(MQClientException.class); + } catch (MQClientException e) { + assertThat(e).hasMessageContaining("Long polling mode, the consumer consumerTimeoutMillisWhenSuspend must greater than brokerSuspendMaxTimeMillis"); + } finally { + litePullConsumer.shutdown(); + } + } private void initDefaultLitePullConsumer(DefaultLitePullConsumer litePullConsumer) throws Exception { @@ -310,17 +449,6 @@ public class DefaultLitePullConsumerTest { DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); litePullConsumer.subscribe(topic, "*"); - - litePullConsumer.start(); - initDefaultLitePullConsumer(litePullConsumer); - return litePullConsumer; - } - - private DefaultLitePullConsumer createBroadcastLitePullConsumer() throws Exception { - DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); - litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); - litePullConsumer.setMessageModel(MessageModel.BROADCASTING); - litePullConsumer.subscribe(topic, "*"); litePullConsumer.start(); initDefaultLitePullConsumer(litePullConsumer); return litePullConsumer; @@ -339,6 +467,24 @@ public class DefaultLitePullConsumerTest { return litePullConsumer; } + private DefaultLitePullConsumer createBroadcastLitePullConsumer() throws Exception { + DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer(consumerGroup + System.currentTimeMillis()); + litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); + litePullConsumer.setMessageModel(MessageModel.BROADCASTING); + litePullConsumer.subscribe(topic, "*"); + litePullConsumer.start(); + initDefaultLitePullConsumer(litePullConsumer); + return litePullConsumer; + } + + private MessageQueue createMessageQueue() { + MessageQueue messageQueue = new MessageQueue(); + messageQueue.setBrokerName(brokerName); + messageQueue.setQueueId(0); + messageQueue.setTopic(topic); + return messageQueue; + } + private PullResultExt createPullResult(PullMessageRequestHeader requestHeader, PullStatus pullStatus, List messageExtList) throws Exception { ByteArrayOutputStream outputStream = new ByteArrayOutputStream();