diff --git a/.gitignore b/.gitignore index c20f4bf768..4ee7621073 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ bazel-out bazel-bin bazel-rocketmq bazel-testlogs -.vscode \ No newline at end of file +.vscode +MODULE.bazel.lock \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000000..15fc5c6e3a --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +############################################################################### +# Bazel now uses Bzlmod by default to manage external dependencies. +# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel. +# +# For more details, please check https://github.com/bazelbuild/bazel/issues/18958 +############################################################################### diff --git a/WORKSPACE b/WORKSPACE index e1f7743302..9b06bc6341 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -112,6 +112,8 @@ maven_install( "com.alipay.sofa:hessian:3.3.6", "io.netty:netty-tcnative-boringssl-static:2.0.48.Final", "org.mockito:mockito-junit-jupiter:4.11.0", + "com.alibaba.fastjson2:fastjson2:2.0.43", + "org.junit.jupiter:junit-jupiter-api:5.9.1", ], fetch_sources = True, repositories = [ diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 66e621e930..c21f9d114c 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -91,6 +91,9 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:org_powermock_powermock_core", "@maven//:io_opentelemetry_opentelemetry_api", + "@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru", + "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:commons_collections_commons_collections", ], ) diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java index 8d95d843db..f43f79b1be 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java @@ -37,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageQueueForC; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; @@ -49,6 +50,7 @@ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequest import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader; import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class Broker2Client { private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -100,13 +102,12 @@ public class Broker2Client { } } - - public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) { + public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) throws RemotingCommandException { return resetOffset(topic, group, timeStamp, isForce, false); } public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce, - boolean isC) { + boolean isC) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -135,8 +136,11 @@ public class Broker2Client { long timeStampOffset; if (timeStamp == -1) { - - timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + try { + timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } } else { timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java index 88e74fd6e5..eddaee706a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java @@ -22,7 +22,6 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; - public class LmqPullRequestHoldService extends PullRequestHoldService { private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -48,8 +47,8 @@ public class LmqPullRequestHoldService extends PullRequestHoldService { } String topic = key.substring(0, idx); int queueId = Integer.parseInt(key.substring(idx + 1)); - final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); try { + final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); this.notifyMessageArriving(topic, queueId, offset); } catch (Throwable e) { LOGGER.error("check hold request failed. topic={}, queueId={}", topic, queueId, e); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java index e8da9d0c47..7dbc9e4fd8 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java @@ -28,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.ConsumeQueueExt; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class PullRequestHoldService extends ServiceThread { private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); @@ -103,8 +104,8 @@ public class PullRequestHoldService extends ServiceThread { if (2 == kArray.length) { String topic = kArray[0]; int queueId = Integer.parseInt(kArray[1]); - final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); try { + final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); this.notifyMessageArriving(topic, queueId, offset); } catch (Throwable e) { log.error( @@ -131,7 +132,12 @@ public class PullRequestHoldService extends ServiceThread { for (PullRequest request : requestList) { long newestOffset = maxOffset; if (newestOffset <= request.getPullFromThisOffset()) { - newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + try { + newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + } catch (ConsumeQueueException e) { + log.error("Failed tp get max offset in queue", e); + continue; + } } if (newestOffset > request.getPullFromThisOffset()) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java index 1930d0dfcb..3ac6528b2a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java @@ -48,6 +48,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SimpleSubscriptionData import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.store.DefaultMessageFilter; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class ConsumerLagCalculator { private final BrokerConfig brokerConfig; @@ -212,22 +213,30 @@ public class ConsumerLagCalculator { CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false); - Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); - if (lag != null) { - result.lag = lag.getObject1(); - result.earliestUnconsumedTimestamp = lag.getObject2(); - } - lagRecorder.accept(result); - - if (info.isPop) { - Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); - - result = new CalculateLagResult(info.group, info.topic, true); - if (retryLag != null) { - result.lag = retryLag.getObject1(); - result.earliestUnconsumedTimestamp = retryLag.getObject2(); + try { + Pair lag = getConsumerLagStats(info.group, info.topic, info.isPop); + if (lag != null) { + result.lag = lag.getObject1(); + result.earliestUnconsumedTimestamp = lag.getObject2(); } lagRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get lag stats", e); + } + + if (info.isPop) { + try { + Pair retryLag = getConsumerLagStats(info.group, info.retryTopic, true); + + result = new CalculateLagResult(info.group, info.topic, true); + if (retryLag != null) { + result.lag = retryLag.getObject1(); + result.earliestUnconsumedTimestamp = retryLag.getObject2(); + } + lagRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get lag stats", e); + } } }); } @@ -235,22 +244,30 @@ public class ConsumerLagCalculator { public void calculateInflight(Consumer inflightRecorder) { processAllGroup(info -> { CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false); - Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop); - if (inFlight != null) { - result.inFlight = inFlight.getObject1(); - result.earliestUnPulledTimestamp = inFlight.getObject2(); - } - inflightRecorder.accept(result); - - if (info.isPop) { - Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true); - - result = new CalculateInflightResult(info.group, info.topic, true); - if (retryInFlight != null) { - result.inFlight = retryInFlight.getObject1(); - result.earliestUnPulledTimestamp = retryInFlight.getObject2(); + try { + Pair inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop); + if (inFlight != null) { + result.inFlight = inFlight.getObject1(); + result.earliestUnPulledTimestamp = inFlight.getObject2(); } inflightRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get inflight message stats", e); + } + + if (info.isPop) { + try { + Pair retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true); + + result = new CalculateInflightResult(info.group, info.topic, true); + if (retryInFlight != null) { + result.inFlight = retryInFlight.getObject1(); + result.earliestUnPulledTimestamp = retryInFlight.getObject2(); + } + inflightRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get inflight message stats", e); + } } }); } @@ -259,20 +276,28 @@ public class ConsumerLagCalculator { processAllGroup(info -> { CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false); - result.available = getAvailableMsgCount(info.group, info.topic, info.isPop); - availableRecorder.accept(result); + try { + result.available = getAvailableMsgCount(info.group, info.topic, info.isPop); + availableRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get available message count", e); + } + if (info.isPop) { - long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true); - - result = new CalculateAvailableResult(info.group, info.topic, true); - result.available = retryAvailable; - availableRecorder.accept(result); + try { + long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true); + result = new CalculateAvailableResult(info.group, info.topic, true); + result.available = retryAvailable; + availableRecorder.accept(result); + } catch (ConsumeQueueException e) { + LOGGER.error("Failed to get available message count", e); + } } }); } - public Pair getConsumerLagStats(String group, String topic, boolean isPop) { + public Pair getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; long earliestUnconsumedTimestamp = Long.MAX_VALUE; @@ -298,7 +323,8 @@ public class ConsumerLagCalculator { return new Pair<>(total, earliestUnconsumedTimestamp); } - public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop) { + public Pair getConsumerLagStats(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId); if (brokerOffset < 0) { brokerOffset = 0; @@ -329,7 +355,7 @@ public class ConsumerLagCalculator { return new Pair<>(lag, consumerStoreTimeStamp); } - public Pair getInFlightMsgStats(String group, String topic, boolean isPop) { + public Pair getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; long earliestUnPulledTimestamp = Long.MAX_VALUE; @@ -355,7 +381,8 @@ public class ConsumerLagCalculator { return new Pair<>(total, earliestUnPulledTimestamp); } - public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) { + public Pair getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { if (isPop) { long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId); long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId); @@ -384,7 +411,7 @@ public class ConsumerLagCalculator { return new Pair<>(inflight, pullStoreTimeStamp); } - public long getAvailableMsgCount(String group, String topic, boolean isPop) { + public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException { long total = 0L; if (group == null || topic == null) { @@ -403,7 +430,8 @@ public class ConsumerLagCalculator { return total; } - public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) { + public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) + throws ConsumeQueueException { long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId); if (brokerOffset < 0) { brokerOffset = 0; diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java index 2de220da16..6e87cb0b69 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java @@ -39,8 +39,11 @@ import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.metrics.NopLongCounter; import org.apache.rocketmq.common.metrics.NopLongHistogram; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC; @@ -57,6 +60,7 @@ import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_QUEUE_ import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE; public class PopMetricsManager { + private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class); public static Supplier attributesBuilderSupplier; private static LongHistogram popBufferScanTimeConsume = new NopLongHistogram(); @@ -138,9 +142,13 @@ public class PopMetricsManager { ObservableLongMeasurement measurement) { PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices(); for (PopReviveService popReviveService : popReviveServices) { - measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder() - .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) - .build()); + try { + measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder() + .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) + .build()); + } catch (ConsumeQueueException e) { + log.error("Failed to get revive behind duration", e); + } } } @@ -148,9 +156,13 @@ public class PopMetricsManager { ObservableLongMeasurement measurement) { PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices(); for (PopReviveService popReviveService : popReviveServices) { - measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder() - .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) - .build()); + try { + measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder() + .put(LABEL_QUEUE_ID, popReviveService.getQueueId()) + .build()); + } catch (ConsumeQueueException e) { + log.error("Failed to get revive behind message count", e); + } } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java index 9896735dd1..79bb0c771d 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java @@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.store.exception.ConsumeQueueException; /** * manage the offset of broadcast. @@ -72,7 +73,7 @@ public class BroadcastOffsetManager extends ServiceThread { * @return -1 means no init offset, use the queueOffset in pullRequestHeader */ public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset, - boolean fromProxy) { + boolean fromProxy) throws ConsumeQueueException { BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId)); if (broadcastOffsetData == null) { @@ -84,29 +85,26 @@ public class BroadcastOffsetManager extends ServiceThread { } final AtomicLong offset = new AtomicLong(-1L); - broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdK, offsetStore) -> { - if (offsetStore == null) { - offsetStore = new BroadcastTimedOffsetStore(fromProxy); - } - - if (offsetStore.fromProxy && requestOffset < 0) { - // when from proxy and requestOffset is -1 - // means proxy need a init offset to pull message - offset.set(getOffset(offsetStore, topic, groupId, queueId)); - return offsetStore; - } - - if (offsetStore.fromProxy == fromProxy) { - return offsetStore; - } + BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId); + if (offsetStore == null) { + offsetStore = new BroadcastTimedOffsetStore(fromProxy); + broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore); + } + if (offsetStore.fromProxy && requestOffset < 0) { + // when from proxy and requestOffset is -1 + // means proxy need a init offset to pull message offset.set(getOffset(offsetStore, topic, groupId, queueId)); - return offsetStore; - }); + } else { + if (offsetStore.fromProxy != fromProxy) { + offset.set(getOffset(offsetStore, topic, groupId, queueId)); + } + } return offset.get(); } - private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) { + private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) + throws ConsumeQueueException { long storeOffset = -1; if (offsetStore != null) { storeOffset = offsetStore.offsetStore.readOffset(queueId); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java index dc1b1b53a3..043ef13f5a 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java @@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSON; import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.util.BitSet; +import java.nio.charset.StandardCharsets; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.KeyBuilder; @@ -30,7 +31,6 @@ import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -45,6 +45,7 @@ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; @@ -134,7 +135,12 @@ public class AckMessageProcessor implements NettyRequestProcessor { } long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset", e); + } if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d", requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset); @@ -166,7 +172,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { } private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck, - final RemotingCommand response, final Channel channel, String brokerName) { + final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException { String[] extraInfo; String consumeGroup, topic; int qId, rqId; @@ -206,7 +212,12 @@ public class AckMessageProcessor implements NettyRequestProcessor { invisibleTime = batchAck.getInvisibleTime(); long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (minOffset == -1 || maxOffset == -1) { POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck); return; @@ -254,7 +265,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(rqId); if (ackMsg instanceof BatchAckMsg) { msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java index 80f3f44fac..aa962513df 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -215,6 +215,7 @@ import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.RocksDBMessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; @@ -1341,8 +1342,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); - final GetMaxOffsetRequestHeader requestHeader = - (GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); + final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader); RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext); @@ -1350,10 +1350,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return rewriteResult; } - long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - - responseHeader.setOffset(offset); - + try { + long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + responseHeader.setOffset(offset); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } response.setCode(ResponseCode.SUCCESS); response.setRemark(null); return response; @@ -1484,7 +1486,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return response; } - private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) { + private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); HashMap runtimeInfo = this.prepareRuntimeInfo(); @@ -1686,8 +1689,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return response; } - - private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) { + private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) + throws ConsumeQueueException { String topic = topicConfig.getTopicName(); for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) { if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) { @@ -1761,8 +1764,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetTopicStatsInfoRequestHeader requestHeader = - (GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); + final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); final String topic = requestHeader.getTopic(); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -1775,39 +1777,45 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { TopicStatsTable topicStatsTable = new TopicStatsTable(); int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums()); - for (int i = 0; i < maxQueueNums; i++) { - MessageQueue mq = new MessageQueue(); - mq.setTopic(topic); - mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); - mq.setQueueId(i); + try { + for (int i = 0; i < maxQueueNums; i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); - TopicOffset topicOffset = new TopicOffset(); - long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i); - if (min < 0) { - min = 0; + TopicOffset topicOffset = new TopicOffset(); + long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i); + if (min < 0) { + min = 0; + } + + long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + if (max < 0) { + max = 0; + } + + long timestamp = 0; + if (max > 0) { + timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); + } + + topicOffset.setMinOffset(min); + topicOffset.setMaxOffset(max); + topicOffset.setLastUpdateTimestamp(timestamp); + + topicStatsTable.getOffsetTable().put(mq, topicOffset); } - long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); - if (max < 0) { - max = 0; - } - - long timestamp = 0; - if (max > 0) { - timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); - } - - topicOffset.setMinOffset(min); - topicOffset.setMaxOffset(max); - topicOffset.setLastUpdateTimestamp(timestamp); - - topicStatsTable.getOffsetTable().put(mq, topicOffset); + byte[] body = topicStatsTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); } - byte[] body = topicStatsTable.encode(); - response.setBody(body); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); return response; } @@ -1907,93 +1915,96 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - final GetConsumeStatsRequestHeader requestHeader = - (GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); + try { + final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); + ConsumeStats consumeStats = new ConsumeStats(); - ConsumeStats consumeStats = new ConsumeStats(); - - Set topics = new HashSet<>(); - if (UtilAll.isBlank(requestHeader.getTopic())) { - topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup()); - } else { - topics.add(requestHeader.getTopic()); - } - - for (String topic : topics) { - TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); - if (null == topicConfig) { - LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic); - continue; + Set topics = new HashSet<>(); + if (UtilAll.isBlank(requestHeader.getTopic())) { + topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup()); + } else { + topics.add(requestHeader.getTopic()); } - TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic); - - { - SubscriptionData findSubscriptionData = - this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); - - if (null == findSubscriptionData - && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) { - LOGGER.warn( - "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, " - + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup()); + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic); continue; } - } - for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { - MessageQueue mq = new MessageQueue(); - mq.setTopic(topic); - mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); - mq.setQueueId(i); + TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic); - OffsetWrapper offsetWrapper = new OffsetWrapper(); + { + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic); - long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); - if (brokerOffset < 0) { - brokerOffset = 0; - } - - long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset( - requestHeader.getConsumerGroup(), topic, i); - - // the consumerOffset cannot be zero for static topic because of the "double read check" strategy - // just remain the logic for dynamic topic - // maybe we should remove it in the future - if (mappingDetail == null) { - if (consumerOffset < 0) { - consumerOffset = 0; + if (null == findSubscriptionData + && this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) { + LOGGER.warn( + "AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, " + + "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup()); + continue; } } - long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset( - requestHeader.getConsumerGroup(), topic, i); + for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); - offsetWrapper.setBrokerOffset(brokerOffset); - offsetWrapper.setConsumerOffset(consumerOffset); - offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset)); + OffsetWrapper offsetWrapper = new OffsetWrapper(); - long timeOffset = consumerOffset - 1; - if (timeOffset >= 0) { - long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); - if (lastTimestamp > 0) { - offsetWrapper.setLastTimestamp(lastTimestamp); + long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + if (brokerOffset < 0) { + brokerOffset = 0; } + + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getConsumerGroup(), topic, i); + + // the consumerOffset cannot be zero for static topic because of the "double read check" strategy + // just remain the logic for dynamic topic + // maybe we should remove it in the future + if (mappingDetail == null) { + if (consumerOffset < 0) { + consumerOffset = 0; + } + } + + long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset( + requestHeader.getConsumerGroup(), topic, i); + + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset)); + + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } + } + + consumeStats.getOffsetTable().put(mq, offsetWrapper); } - consumeStats.getOffsetTable().put(mq, offsetWrapper); + double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic); + + consumeTps += consumeStats.getConsumeTps(); + consumeStats.setConsumeTps(consumeTps); } - double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic); - - consumeTps += consumeStats.getConsumeTps(); - consumeStats.setConsumeTps(consumeTps); + byte[] body = consumeStats.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); } - - byte[] body = consumeStats.encode(); - response.setBody(body); - response.setCode(ResponseCode.SUCCESS); - response.setRemark(null); return response; } @@ -2108,7 +2119,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { requestHeader.getTimestamp(), requestHeader.isForce(), isC); } - private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) { + private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException { if (timestamp < 0) { return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); } else { @@ -2155,25 +2166,31 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return response; } - if (queueId >= 0) { - if (null != offset && -1 != offset) { - long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); - long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); - if (min >= 0 && offset < min || offset > max + 1) { - response.setCode(ResponseCode.SYSTEM_ERROR); - response.setRemark( - String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max)); - return response; + try { + if (queueId >= 0) { + if (null != offset && -1 != offset) { + long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); + long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + if (min >= 0 && offset < min || offset > max + 1) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark( + String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max)); + return response; + } + } else { + offset = searchOffsetByTimestamp(topic, queueId, timestamp); } + queueOffsetMap.put(queueId, offset); } else { - offset = searchOffsetByTimestamp(topic, queueId, timestamp); - } - queueOffsetMap.put(queueId, offset); - } else { - for (int index = 0; index < topicConfig.getReadQueueNums(); index++) { - offset = searchOffsetByTimestamp(topic, index, timestamp); - queueOffsetMap.put(index, offset); + for (int index = 0; index < topicConfig.getReadQueueNums(); index++) { + offset = searchOffsetByTimestamp(topic, index, timestamp); + queueOffsetMap.put(index, offset); + } } + } catch (ConsumeQueueException e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(e.getMessage()); + return response; } if (queueOffsetMap.isEmpty()) { @@ -2280,8 +2297,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - QueryConsumeTimeSpanRequestHeader requestHeader = - (QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); + QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); final String topic = requestHeader.getTopic(); TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); @@ -2303,7 +2319,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i); timeSpan.setMinTimeStamp(minTime); - long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + long max; + try { + max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1); timeSpan.setMaxTimeStamp(maxTime); @@ -2317,7 +2338,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { } timeSpan.setConsumeTimeStamp(consumeTime); - long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i); + long maxBrokerOffset; + try { + maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (consumerOffset < maxBrokerOffset) { long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset); timeSpan.setDelayTime(System.currentTimeMillis() - nextTime); @@ -2552,8 +2578,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request) throws RemotingCommandException { final RemotingCommand response = RemotingCommand.createResponseCommand(null); - GetConsumeStatsInBrokerHeader requestHeader = - (GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class); + GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class); boolean isOrder = requestHeader.isOrder(); ConcurrentMap subscriptionGroups = brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable(); @@ -2599,7 +2624,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); mq.setQueueId(i); OffsetWrapper offsetWrapper = new OffsetWrapper(); - long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + long brokerOffset; + try { + brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset", e); + } if (brokerOffset < 0) { brokerOffset = 0; } @@ -2643,7 +2673,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { return response; } - private HashMap prepareRuntimeInfo() { + private HashMap prepareRuntimeInfo() throws RemotingCommandException { HashMap runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo(); for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) { @@ -2652,7 +2682,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { } } - this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo); + try { + this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max offset in queue", e); + } runtimeInfo.put("brokerActive", String.valueOf(this.brokerController.isSpecialServiceRunning())); runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION)); runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION)); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index af3b8ae6f0..d29ff2a55b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -21,6 +21,7 @@ import io.netty.channel.Channel; import io.netty.channel.ChannelHandlerContext; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.nio.charset.StandardCharsets; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.metrics.PopMetricsManager; import org.apache.rocketmq.common.PopAckConstants; @@ -30,7 +31,6 @@ import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -43,6 +43,7 @@ import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHe import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader; import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil; import org.apache.rocketmq.store.PutMessageStatus; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -120,7 +121,12 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { return CompletableFuture.completedFuture(response); } long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); - long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + long maxOffset; + try { + maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId()); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to get max consume offset", e); + } if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) { response.setCode(ResponseCode.NO_MESSAGE); return CompletableFuture.completedFuture(response); @@ -201,7 +207,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { } msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(rqId); msgInner.setTags(PopAckConstants.ACK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); @@ -244,7 +250,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { ck.addDiff(0); ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo)); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java index c82725fe1e..75c77b6d79 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java @@ -41,6 +41,7 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode; import org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader; import org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader; import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; public class NotificationProcessor implements NettyRequestProcessor { private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME); @@ -169,13 +170,15 @@ public class NotificationProcessor implements NettyRequestProcessor { return response; } - private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) { + private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) + throws RemotingCommandException { boolean hasMsg; TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName); return hasMsgFromTopic(topicConfig, randomQ, requestHeader); } - private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) { + private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) + throws RemotingCommandException { boolean hasMsg; if (topicConfig != null) { for (int i = 0; i < topicConfig.getReadQueueNums(); i++) { @@ -189,15 +192,19 @@ public class NotificationProcessor implements NettyRequestProcessor { return false; } - private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) { + private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) throws RemotingCommandException { if (Boolean.TRUE.equals(requestHeader.getOrder())) { if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) { return false; } } long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId); - long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset; - return restNum > 0; + try { + long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset; + return restNum > 0; + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed tp get max offset in queue", e); + } } private long getPopOffset(String topic, String cid, int queueId) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java index 2c0a1cd54a..8473e3a286 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java @@ -51,6 +51,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM; @@ -229,13 +230,18 @@ public class PeekMessageProcessor implements NettyRequestProcessor { private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult, PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel, - long popTime) { + long popTime) throws RemotingCommandException { String topic = isRetry ? KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2()) : requestHeader.getTopic(); GetMessageResult getMessageTmpResult; long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId); - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + } catch (ConsumeQueueException e) { + LOG.error("Failed to get max offset in queue. topic={}, queue-id={}", topic, queueId, e); + throw new RemotingCommandException("Failed to get max offset in queue", e); + } if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) { return restNum; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 2d76c5a3ca..07bc0ac07b 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -24,6 +24,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.FileRegion; import io.opentelemetry.api.common.Attributes; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -62,7 +63,6 @@ import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.common.topic.TopicValidator; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; @@ -83,6 +83,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -167,13 +168,14 @@ public class PopMessageProcessor implements NettyRequestProcessor { return popLongPollingService.getPollingMap(); } - public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) { + public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException { this.notifyLongPollingRequestIfNeed( topic, group, queueId, null, 0L, null, null); } public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId, - Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map properties) { + Long tagsCode, long msgStoreTime, byte[] filterBitMap, + Map properties) throws ConsumeQueueException { long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId); long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId); long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); @@ -217,8 +219,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); - final PopMessageRequestHeader requestHeader = - (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true); + final PopMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true); StringBuilder startOffsetInfo = new StringBuilder(64); StringBuilder msgOffsetInfo = new StringBuilder(64); StringBuilder orderCountInfo = null; @@ -531,20 +532,37 @@ public class PopMessageProcessor implements NettyRequestProcessor { String lockKey = topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId; boolean isOrder = requestHeader.isOrder(); - long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), - false, lockKey, false); + long offset; + try { + offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(), + false, lockKey, false); + } catch (ConsumeQueueException e) { + CompletableFuture failure = new CompletableFuture<>(); + failure.completeExceptionally(e); + return failure; + } + CompletableFuture future = new CompletableFuture<>(); if (!queueLockManager.tryLock(lockKey)) { - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; - future.complete(restNum); + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + future.complete(restNum); + } catch (ConsumeQueueException e) { + future.completeExceptionally(e); + } return future; } future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey)); if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) { - POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId); - restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; - future.complete(restNum); + POP_LOGGER.warn("Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}", + topic, requestHeader.getConsumerGroup(), queueId); + try { + restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum; + future.complete(restNum); + } catch (ConsumeQueueException e) { + future.completeExceptionally(e); + } return future; } @@ -610,7 +628,11 @@ public class PopMessageProcessor implements NettyRequestProcessor { return CompletableFuture.completedFuture(result); }).thenApply(result -> { if (result == null) { - atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get()); + try { + atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get()); + } catch (ConsumeQueueException e) { + POP_LOGGER.error("Failed to get max offset in queue", e); + } return atomicRestNum.get(); } if (!result.getMessageMapedList().isEmpty()) { @@ -710,7 +732,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { } private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey, - boolean checkResetOffset) { + boolean checkResetOffset) throws ConsumeQueueException { long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId); if (offset < 0) { @@ -732,7 +754,8 @@ public class PopMessageProcessor implements NettyRequestProcessor { } } - private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) { + private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) + throws ConsumeQueueException { long offset; if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId); @@ -761,7 +784,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); msgInner.setTopic(reviveTopic); - msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8)); msgInner.setQueueId(reviveQid); msgInner.setTags(PopAckConstants.CK_TAG); msgInner.setBornTimestamp(System.currentTimeMillis()); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java index 4b141d2910..f27934efdf 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java @@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor; import com.alibaba.fastjson.JSON; import io.opentelemetry.api.common.Attributes; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -54,6 +55,7 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.AppendMessageStatus; import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.PutMessageResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.pop.AckMsg; import org.apache.rocketmq.store.pop.BatchAckMsg; import org.apache.rocketmq.store.pop.PopCheckPoint; @@ -260,10 +262,14 @@ public class PopReviveService extends ServiceThread { getMessageResult.getMaxOffset(), foundList); } else { - long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); - if (maxQueueOffset > offset) { - POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}", - topic, group, offset, maxQueueOffset); + try { + long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId); + if (maxQueueOffset > offset) { + POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}", + topic, group, offset, maxQueueOffset); + } + } catch (ConsumeQueueException e) { + POP_LOGGER.error("Failed to get max offset in queue", e); } return null; } @@ -364,7 +370,7 @@ public class PopReviveService extends ServiceThread { firstRt = point.getReviveTime(); } } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -388,7 +394,7 @@ public class PopReviveService extends ServiceThread { } } } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { - String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8); if (brokerController.getBrokerConfig().isEnablePopLog()) { POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); } @@ -594,7 +600,7 @@ public class PopReviveService extends ServiceThread { brokerController.getMessageStore().putMessage(ckMsg); } - public long getReviveBehindMillis() { + public long getReviveBehindMillis() throws ConsumeQueueException { if (currentReviveMessageTimestamp <= 0) { return 0; } @@ -605,7 +611,7 @@ public class PopReviveService extends ServiceThread { return 0; } - public long getReviveBehindMessages() { + public long getReviveBehindMessages() throws ConsumeQueueException { if (currentReviveMessageTimestamp <= 0) { return 0; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java index 6dd8b30047..2ad2c9e93e 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java @@ -73,6 +73,7 @@ import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.MessageFilter; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.config.BrokerRole; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.stats.BrokerStatsManager; import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse; @@ -298,7 +299,8 @@ public class PullMessageProcessor implements NettyRequestProcessor { return false; } - private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, boolean brokerAllowFlowCtrSuspend) + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, + boolean brokerAllowFlowCtrSuspend) throws RemotingCommandException { final long beginTimeMills = this.brokerController.getMessageStore().now(); RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); @@ -489,7 +491,7 @@ public class PullMessageProcessor implements NettyRequestProcessor { final MessageStore messageStore = brokerController.getMessageStore(); if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) { - DefaultMessageStore defaultMessageStore = (DefaultMessageStore)this.brokerController.getMessageStore(); + DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore(); boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup()); if (cgNeedColdDataFlowCtr) { boolean isMsgLogicCold = defaultMessageStore.getCommitLog() @@ -526,7 +528,11 @@ public class PullMessageProcessor implements NettyRequestProcessor { getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET); getMessageResult.setNextBeginOffset(resetOffset); getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId)); - getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId)); + try { + getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId)); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed tp get max offset in queue", e); + } getMessageResult.setSuggestPullingFromSlave(false); } else { long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel); @@ -589,12 +595,13 @@ public class PullMessageProcessor implements NettyRequestProcessor { /** * Composes the header of the response message to be sent back to the client - * @param requestHeader - the header of the request message - * @param getMessageResult - the result of the GetMessage request - * @param topicSysFlag - the system flag of the topic + * + * @param requestHeader - the header of the request message + * @param getMessageResult - the result of the GetMessage request + * @param topicSysFlag - the system flag of the topic * @param subscriptionGroupConfig - configuration of the subscription group - * @param response - the response message to be sent back to the client - * @param clientAddress - the address of the client + * @param response - the response message to be sent back to the client + * @param clientAddress - the address of the client */ protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult, int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response, @@ -855,7 +862,7 @@ public class PullMessageProcessor implements NettyRequestProcessor { * When pull request is not broadcast or not return -1 */ protected long queryBroadcastPullInitOffset(String topic, String group, int queueId, - PullMessageRequestHeader requestHeader, Channel channel) { + PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException { if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) { return -1L; @@ -877,8 +884,12 @@ public class PullMessageProcessor implements NettyRequestProcessor { clientId = clientChannelInfo.getClientId(); } - return this.brokerController.getBroadcastOffsetManager() - .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast); + try { + return this.brokerController.getBroadcastOffsetManager() + .queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast); + } catch (ConsumeQueueException e) { + throw new RemotingCommandException("Failed to query initial offset", e); + } } return -1L; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java index e13b36df91..70184e8a62 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java @@ -53,6 +53,7 @@ import org.apache.rocketmq.remoting.protocol.DataVersion; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.queue.ReferredIterator; @@ -103,7 +104,7 @@ public class ScheduleMessageService extends ConfigManager { return delayLevel - 1; } - public void buildRunningStats(HashMap stats) { + public void buildRunningStats(HashMap stats) throws ConsumeQueueException { for (Map.Entry next : this.offsetTable.entrySet()) { int queueId = delayLevel2QueueId(next.getKey()); long delayOffset = next.getValue(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java index 865e7b608e..7e16d329e1 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java @@ -29,6 +29,7 @@ import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.protocol.RemotingCommand; import org.apache.rocketmq.remoting.protocol.RemotingSerializable; import org.apache.rocketmq.remoting.protocol.ResponseCode; @@ -125,14 +126,14 @@ public class Broker2ClientTest { } @Test - public void testResetOffsetNoTopicConfig() { + public void testResetOffsetNoTopicConfig() throws RemotingCommandException { when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null); RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce); assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode()); } @Test - public void testResetOffsetNoConsumerGroupInfo() { + public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException { TopicConfig topicConfig = mock(TopicConfig.class); when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); when(topicConfig.getWriteQueueNums()).thenReturn(1); @@ -142,7 +143,7 @@ public class Broker2ClientTest { } @Test - public void testResetOffset() { + public void testResetOffset() throws RemotingCommandException { TopicConfig topicConfig = mock(TopicConfig.class); when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig); when(topicConfig.getWriteQueueNums()).thenReturn(1); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java index 9dc00f9d6b..ad5af92646 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.broker.client.ClientChannelInfo; import org.apache.rocketmq.broker.client.ConsumerManager; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -51,7 +52,7 @@ public class BroadcastOffsetManagerTest { private BroadcastOffsetManager broadcastOffsetManager; @Before - public void before() { + public void before() throws ConsumeQueueException { brokerConfig.setEnableBroadcastOffsetStore(true); brokerConfig.setBroadcastOffsetExpireSecond(1); brokerConfig.setBroadcastOffsetExpireMaxSecond(5); @@ -84,7 +85,7 @@ public class BroadcastOffsetManagerTest { } @Test - public void testBroadcastOffsetSwitch() { + public void testBroadcastOffsetSwitch() throws ConsumeQueueException { // client1 connect to broker onlineClientIdSet.add("client1"); long offset = broadcastOffsetManager.queryInitOffset("group", "topic", 0, "client1", 0, false); @@ -160,4 +161,4 @@ public class BroadcastOffsetManagerTest { return broadcastOffsetManager.offsetStoreMap.isEmpty(); }); } -} \ No newline at end of file +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java index 93689efa58..1fdf454d5e 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java @@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.broker.processor.PopMessageProcessor; import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.assertj.core.util.Lists; import org.junit.Before; import org.junit.Test; @@ -50,7 +51,7 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest { private final BrokerController brokerController = mock(BrokerController.class); @Before - public void before() { + public void before() throws ConsumeQueueException { notified = new AtomicBoolean(false); brokerConfig.setEnableNotifyAfterPopOrderLockRelease(true); when(brokerController.getBrokerConfig()).thenReturn(brokerConfig); @@ -175,4 +176,4 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest { await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get); assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty()); } -} \ No newline at end of file +} diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java index c0afb46c33..757b01b63f 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java @@ -47,6 +47,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.PutMessageStatus; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -93,7 +94,7 @@ public class AckMessageProcessorTest { private static final long MAX_OFFSET_IN_QUEUE = 999; @Before - public void init() throws IllegalAccessException, NoSuchFieldException { + public void init() throws IllegalAccessException, NoSuchFieldException, ConsumeQueueException { clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0); brokerController.setMessageStore(messageStore); Field field = BrokerController.class.getDeclaredField("broker2Client"); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java index 8a2ce8a2ba..fdb0690e5d 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.store.GetMessageResult; import org.apache.rocketmq.store.GetMessageStatus; import org.apache.rocketmq.store.SelectMappedBufferResult; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.logfile.DefaultMappedFile; import org.junit.Assert; import org.junit.Before; @@ -182,7 +183,7 @@ public class PopMessageProcessorTest { } @Test - public void testGetInitOffset_normalTopic() throws RemotingCommandException { + public void testGetInitOffset_normalTopic() throws RemotingCommandException, ConsumeQueueException { long maxOffset = 999L; when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig()); when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset); diff --git a/client/BUILD.bazel b/client/BUILD.bazel index 9b6fbc298c..b93f3d9099 100644 --- a/client/BUILD.bazel +++ b/client/BUILD.bazel @@ -33,6 +33,7 @@ java_library( "@maven//:commons_collections_commons_collections", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", + "@maven//:com_google_guava_guava", ], ) diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 0e5571eb13..716d081ef4 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -1199,9 +1199,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdo messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) { // process LMQ String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); long offset = Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]); // LMQ topic has only 1 queue, which queue id is 0 queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID); @@ -1264,9 +1264,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdo && StringUtils.isNotEmpty(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) { // process LMQ String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET) - .split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + .split(MixAll.LMQ_DISPATCH_SEPARATOR); // LMQ topic has only 1 queue, which queue id is 0 key = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID); sortMap.putIfAbsent(key, new ArrayList<>(4)); diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java index e311e0c9b8..81dc5883fb 100644 --- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java @@ -636,8 +636,8 @@ public class MQClientAPIImplTest { final int invisibleTime = 10 * 1000; final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1"; final String lmqTopic2 = MixAll.LMQ_PREFIX + "lmq2"; - final String multiDispatch = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, lmqTopic, lmqTopic2); - final String multiOffset = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, "0", "0"); + final String multiDispatch = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, lmqTopic, lmqTopic2); + final String multiOffset = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, "0", "0"); doAnswer((Answer) mock -> { InvokeCallback callback = mock.getArgument(3); RemotingCommand request = mock.getArgument(1); diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java index efb115509a..39933038ba 100644 --- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java +++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java @@ -48,6 +48,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.annotation.ImportantField; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.help.FAQUrl; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.utils.IOTinyUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; @@ -99,8 +100,8 @@ public class MixAll { public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml"; public static final String REPLY_MESSAGE_FLAG = "reply"; public static final String LMQ_PREFIX = "%LMQ%"; - public static final long LMQ_QUEUE_ID = 0; - public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ","; + public static final int LMQ_QUEUE_ID = 0; + public static final String LMQ_DISPATCH_SEPARATOR = ","; public static final String REQ_T = "ReqT"; public static final String ROCKETMQ_ZONE_ENV = "ROCKETMQ_ZONE"; public static final String ROCKETMQ_ZONE_PROPERTY = "rocketmq.zone"; @@ -524,4 +525,10 @@ public class MixAll { } return false; } + + public static boolean topicAllowsLMQ(String topic) { + return !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) + && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java index 95dc8b9800..96195d5309 100644 --- a/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java +++ b/common/src/main/java/org/apache/rocketmq/common/ServiceThread.java @@ -24,7 +24,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; public abstract class ServiceThread implements Runnable { - private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); + protected static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME); private static final long JOIN_TIME = 90 * 1000; diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java index 95d5119cfc..a4ba35bd5a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java +++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java @@ -16,8 +16,8 @@ */ package org.apache.rocketmq.common.config; +import com.google.common.base.Strings; import java.io.File; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.UtilAll; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.BloomFilter; @@ -110,12 +110,26 @@ public class ConfigHelper { } public static String getDBLogDir() { - String rootPath = System.getProperty("user.home"); - if (StringUtils.isEmpty(rootPath)) { - return ""; + String[] rootPaths = new String[] { + System.getProperty("user.home"), + System.getProperty("java.io.tmpdir"), + File.separator + "data" + }; + for (String rootPath : rootPaths) { + // Refer bazel test encyclopedia: https://bazel.build/reference/test-encyclopedia + // Not all directories is available + if (Strings.isNullOrEmpty(rootPath)) { + continue; + } + File rootPathFile = new File(rootPath); + if (!rootPathFile.exists() || !rootPathFile.canWrite()) { + continue; + } + String logDirectory = rootPath + File.separator + "logs" + File.separator + "rocketmqlogs"; + // Create directories recursively. + UtilAll.ensureDirOK(logDirectory); + return logDirectory; } - rootPath = rootPath + File.separator + "logs"; - UtilAll.ensureDirOK(rootPath); - return rootPath + File.separator + "rocketmqlogs" + File.separator; + throw new RuntimeException("Failed to get log directory"); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java index 147f23f123..e4f02e2c9b 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java @@ -16,8 +16,11 @@ */ package org.apache.rocketmq.common.message; +import com.google.common.base.Strings; import java.nio.ByteBuffer; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.common.utils.MessageUtils; @@ -41,7 +44,7 @@ public class MessageExtBrokerInner extends MessageExt { } public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { - if (null == tags || tags.length() == 0) { return 0; } + if (Strings.isNullOrEmpty(tags)) { return 0; } return tags.hashCode(); } @@ -102,4 +105,9 @@ public class MessageExtBrokerInner extends MessageExt { public void setEncodeCompleted(boolean encodeCompleted) { this.encodeCompleted = encodeCompleted; } + + public boolean needDispatchLMQ() { + return StringUtils.isNoneBlank(getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) + && MixAll.topicAllowsLMQ(getTopic()); + } } diff --git a/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java new file mode 100644 index 0000000000..1fcc967d67 --- /dev/null +++ b/common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.config; + +import org.junit.Test; + +public class ConfigHelperTest { + + @Test + public void testGetDBLogDir() { + // Should not raise exception. + ConfigHelper.getDBLogDir(); + } + +} diff --git a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java index 5fee948028..da6ad920f3 100644 --- a/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java +++ b/example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java @@ -49,7 +49,7 @@ public class LMQProducer { Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); msg.setKeys("Key" + i); msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */, - String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); + String.join(MixAll.LMQ_DISPATCH_SEPARATOR, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */); SendResult sendResult = producer.send(msg); System.out.printf("%s%n", sendResult); } catch (Exception e) { diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java index 6c7327f258..0e35acc01f 100644 --- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java +++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java @@ -49,7 +49,7 @@ public class FileRegionEncoderTest { random.nextBytes(data); write(file, data); FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength); - Assert.assertEquals(0, fileRegion.transfered()); + Assert.assertEquals(0, fileRegion.transferred()); Assert.assertEquals(dataLength, fileRegion.count()); Assert.assertTrue(channel.writeOutbound(fileRegion)); ByteBuf out = (ByteBuf) channel.readOutbound(); @@ -77,4 +77,4 @@ public class FileRegionEncoderTest { } } } -} \ No newline at end of file +} diff --git a/store/BUILD.bazel b/store/BUILD.bazel index 8364a239c9..98f90a577c 100644 --- a/store/BUILD.bazel +++ b/store/BUILD.bazel @@ -42,6 +42,8 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:com_google_code_findbugs_jsr305", + "@maven//:commons_validator_commons_validator", ], ) @@ -63,6 +65,7 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:org_junit_jupiter_junit_jupiter_api", ], ) diff --git a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java index 4d53f3b7bc..c3534d0611 100644 --- a/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java +++ b/store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java @@ -25,4 +25,5 @@ public enum AppendMessageStatus { MESSAGE_SIZE_EXCEEDED, PROPERTIES_SIZE_EXCEEDED, UNKNOWN_ERROR, + ROCKSDB_ERROR, } diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java index 972e71aadd..153215c98a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.store; +import com.google.common.base.Strings; import java.net.Inet6Address; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -35,7 +36,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import com.sun.jna.NativeLong; import com.sun.jna.Pointer; -import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.SystemClock; @@ -58,10 +58,11 @@ import org.apache.rocketmq.store.MessageExtEncoder.PutMessageThreadLocal; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; import org.apache.rocketmq.store.logfile.MappedFile; -import org.apache.rocketmq.store.queue.MultiDispatchUtils; import org.apache.rocketmq.store.util.LibC; import org.rocksdb.RocksDBException; @@ -104,7 +105,6 @@ public class CommitLog implements Swappable { protected int commitLogSize; private final boolean enabledAppendPropCRC; - protected final MultiDispatch multiDispatch; public CommitLog(final DefaultMessageStore messageStore) { String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog(); @@ -139,8 +139,6 @@ public class CommitLog implements Swappable { this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); this.enabledAppendPropCRC = messageStore.getMessageStoreConfig().isEnabledAppendPropCRC(); - - this.multiDispatch = new MultiDispatch(defaultMessageStore); } public void setFullStorePaths(Set fullStorePaths) { @@ -530,7 +528,7 @@ public class CommitLog implements Swappable { } String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS); - if (tags != null && tags.length() > 0) { + if (!Strings.isNullOrEmpty(tags)) { tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags); } @@ -652,7 +650,7 @@ public class CommitLog implements Swappable { } else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) { return this.confirmOffset; } else { - return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset(); + return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset(); } } @@ -770,8 +768,11 @@ public class CommitLog implements Swappable { } } - // only for rocksdb mode - this.getMessageStore().finishCommitLogDispatch(); + try { + this.getMessageStore().getQueueStore().flush(); + } catch (StoreException e) { + log.error("Failed to flush ConsumeQueueStore", e); + } processOffset += mappedFileOffset; if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { @@ -988,7 +989,7 @@ public class CommitLog implements Swappable { msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer()); PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey); - putMessageLock.lock(); //spin or ReentrantLock ,depending on store config + putMessageLock.lock(); //spin or ReentrantLock, depending on store config try { long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); this.beginTimeInLock = beginLockTimestamp; @@ -1850,7 +1851,16 @@ public class CommitLog implements Swappable { return null; } - multiDispatch.wrapMultiDispatch(msgInner); + try { + LmqDispatch.wrapLmqDispatch(defaultMessageStore, msgInner); + } catch (ConsumeQueueException e) { + if (e.getCause() instanceof RocksDBException) { + log.error("Failed to wrap multi-dispatch", e); + return new AppendMessageResult(AppendMessageStatus.ROCKSDB_ERROR); + } + log.error("Failed to wrap multi-dispatch", e); + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); @@ -1904,7 +1914,7 @@ public class CommitLog implements Swappable { // STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff(); - final boolean isMultiDispatchMsg = CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner); + boolean isMultiDispatchMsg = messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ(); if (isMultiDispatchMsg) { AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner); if (appendMessageResult != null) { @@ -2000,7 +2010,12 @@ public class CommitLog implements Swappable { msgInner.setEncodedBuff(null); if (isMultiDispatchMsg) { - CommitLog.this.multiDispatch.updateMultiQueueOffset(msgInner); + try { + LmqDispatch.updateLmqOffsets(defaultMessageStore, msgInner); + } catch (ConsumeQueueException e) { + // Increase in-memory max offset of the queue should not fail. + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } } return new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgIdSupplier, @@ -2245,11 +2260,6 @@ public class CommitLog implements Swappable { return flushManager; } - public static boolean isMultiDispatchMsg(MessageStoreConfig messageStoreConfig, MessageExtBrokerInner msg) { - return StringUtils.isNotBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) && - MultiDispatchUtils.isNeedHandleMultiDispatch(messageStoreConfig, msg.getTopic()); - } - private boolean isCloseReadAhead() { return !MixAll.isWindows() && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java index eb8af4ab19..b6b9cff538 100644 --- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java @@ -727,8 +727,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { Map prop = request.getPropertiesMap(); String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR); if (queues.length != queueOffsets.length) { log.error("[bug] queues.length!=queueOffsets.length ", request.getTopic()); return; diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java index 8b46c7f5ce..6b8ea0ee8a 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java @@ -93,6 +93,7 @@ import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.dledger.DLedgerCommitLog; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.DefaultHAService; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; @@ -170,7 +171,7 @@ public class DefaultMessageStore implements MessageStore { private RocksDBMessageStore rocksDBMessageStore; - private RandomAccessFile lockFile; + private final RandomAccessFile lockFile; private FileLock lock; @@ -190,7 +191,7 @@ public class DefaultMessageStore implements MessageStore { private volatile long brokerInitMaxOffset = -1L; - private List putMessageHookList = new ArrayList<>(); + private final List putMessageHookList = new ArrayList<>(); private SendMessageBackHook sendMessageBackHook; @@ -203,20 +204,21 @@ public class DefaultMessageStore implements MessageStore { private final ConcurrentLinkedQueue batchDispatchRequestQueue = new ConcurrentLinkedQueue<>(); - private int dispatchRequestOrderlyQueueSize = 16; + private final int dispatchRequestOrderlyQueueSize = 16; private final DispatchRequestOrderlyQueue dispatchRequestOrderlyQueue = new DispatchRequestOrderlyQueue(dispatchRequestOrderlyQueueSize); private long stateMachineVersion = 0L; // this is a unmodifiableMap - private ConcurrentMap topicConfigTable; + private final ConcurrentMap topicConfigTable; private final ScheduledExecutorService scheduledCleanQueueExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, - final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { + final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, + final ConcurrentMap topicConfigTable) throws IOException { this.messageArrivingListener = messageArrivingListener; this.brokerConfig = brokerConfig; this.messageStoreConfig = messageStoreConfig; @@ -438,7 +440,7 @@ public class DefaultMessageStore implements MessageStore { return; } - /** + /* * 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog; * 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go; * 3. Calculate the reput offset according to the consume queue; @@ -458,7 +460,7 @@ public class DefaultMessageStore implements MessageStore { } if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) { maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset(); - /** + /* * This happens in following conditions: * 1. If someone removes all the consumequeue files or the disk get damaged. * 2. Launch a new broker, and copy the commitlog from other brokers. @@ -987,12 +989,12 @@ public class DefaultMessageStore implements MessageStore { } @Override - public long getMaxOffsetInQueue(String topic, int queueId) { + public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException { return getMaxOffsetInQueue(topic, queueId, true); } @Override - public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException { if (committed) { ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId); if (logic != null) { @@ -1378,7 +1380,6 @@ public class DefaultMessageStore implements MessageStore { * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, * consume queue will be created with old offset, then later message with new offset table can not be * dispatched to consume queue. - * @throws RocksDBException only in rocksdb mode */ @Override public int deleteTopics(final Set deleteTopics) { @@ -1748,10 +1749,11 @@ public class DefaultMessageStore implements MessageStore { /** * The ratio val is estimated by the experiment and experience * so that the result is not high accurate for different business + * * @return */ public boolean checkInColdAreaByCommitOffset(long offsetPy, long maxOffsetPy) { - long memory = (long)(StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0)); + long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0)); return (maxOffsetPy - offsetPy) > memory; } @@ -1929,11 +1931,6 @@ public class DefaultMessageStore implements MessageStore { return messageStoreConfig; } - @Override - public void finishCommitLogDispatch() { - // ignore - } - @Override public TransientStorePool getTransientStorePool() { return transientStorePool; @@ -2713,15 +2710,15 @@ public class DefaultMessageStore implements MessageStore { } } - class BatchDispatchRequest { + static class BatchDispatchRequest { - private ByteBuffer byteBuffer; + private final ByteBuffer byteBuffer; - private int position; + private final int position; - private int size; + private final int size; - private long id; + private final long id; public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long id) { this.byteBuffer = byteBuffer; @@ -2731,7 +2728,7 @@ public class DefaultMessageStore implements MessageStore { } } - class DispatchRequestOrderlyQueue { + static class DispatchRequestOrderlyQueue { DispatchRequest[][] buffer; @@ -2907,8 +2904,6 @@ public class DefaultMessageStore implements MessageStore { } finally { result.release(); } - - finishCommitLogDispatch(); } } @@ -2922,8 +2917,8 @@ public class DefaultMessageStore implements MessageStore { if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { return; } - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR); if (queues.length != queueOffsets.length) { return; } @@ -2932,7 +2927,7 @@ public class DefaultMessageStore implements MessageStore { long queueOffset = Long.parseLong(queueOffsets[i]); int queueId = dispatchRequest.getQueueId(); if (DefaultMessageStore.this.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; + queueId = MixAll.LMQ_QUEUE_ID; } DefaultMessageStore.this.messageArrivingListener.arriving( queueName, queueId, queueOffset + 1, dispatchRequest.getTagsCode(), @@ -2972,13 +2967,13 @@ public class DefaultMessageStore implements MessageStore { public MainBatchDispatchRequestService() { batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( - DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), - DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), - 1000 * 60, - TimeUnit.MICROSECONDS, - new LinkedBlockingQueue<>(4096), - new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"), - new ThreadPoolExecutor.AbortPolicy()); + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), + 1000 * 60, + TimeUnit.MICROSECONDS, + new LinkedBlockingQueue<>(4096), + new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"), + new ThreadPoolExecutor.AbortPolicy()); } private void pollBatchDispatchRequest() { @@ -3188,9 +3183,6 @@ public class DefaultMessageStore implements MessageStore { result.release(); } } - - // only for rocksdb mode - finishCommitLogDispatch(); } /** diff --git a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java index 79d006bafc..654760b88c 100644 --- a/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java +++ b/store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java @@ -17,6 +17,9 @@ package org.apache.rocketmq.store; import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageConst; public class DispatchRequest { private final String topic; @@ -228,6 +231,18 @@ public class DispatchRequest { this.offsetId = offsetId; } + public boolean containsLMQ() { + if (!MixAll.topicAllowsLMQ(topic)) { + return false; + } + if (null == propertiesMap || propertiesMap.isEmpty()) { + return false; + } + String lmqNames = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String lmqOffsets = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + return !StringUtils.isBlank(lmqNames) && !StringUtils.isBlank(lmqOffsets); + } + @Override public String toString() { return "DispatchRequest{" + diff --git a/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java new file mode 100644 index 0000000000..2805f51014 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.store.exception.ConsumeQueueException; + +public class LmqDispatch { + private static final short VALUE_OF_EACH_INCREMENT = 1; + + public static void wrapLmqDispatch(MessageStore messageStore, final MessageExtBrokerInner msg) + throws ConsumeQueueException { + String lmqNames = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + Long[] queueOffsets = new Long[queueNames.length]; + if (messageStore.getMessageStoreConfig().isEnableLmq()) { + for (int i = 0; i < queueNames.length; i++) { + if (MixAll.isLmq(queueNames[i])) { + queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(queueNames[i], MixAll.LMQ_QUEUE_ID); + } + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.LMQ_DISPATCH_SEPARATOR)); + msg.removeWaitStorePropertyString(); + } + + public static void updateLmqOffsets(MessageStore messageStore, final MessageExtBrokerInner msgInner) + throws ConsumeQueueException { + String lmqNames = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + for (String queueName : queueNames) { + if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { + messageStore.getQueueStore().increaseLmqOffset(queueName, MixAll.LMQ_QUEUE_ID, VALUE_OF_EACH_INCREMENT); + } + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java index 5c74918d9e..7531c96d11 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java @@ -175,11 +175,11 @@ public class MessageExtEncoder { public PutMessageResult encode(MessageExtBrokerInner msgInner) { this.byteBuf.clear(); - if (CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner)) { + if (messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ()) { return encodeWithoutProperties(msgInner); } - /** + /* * Serialize message */ final byte[] propertiesData = diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java index 814c6d1bfe..5c3984e5b2 100644 --- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; +import javax.annotation.Nonnull; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.SystemClock; @@ -31,6 +32,7 @@ import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; @@ -181,7 +183,7 @@ public interface MessageStore { * @param queueId Queue ID. * @return Maximum offset at present. */ - long getMaxOffsetInQueue(final String topic, final int queueId); + long getMaxOffsetInQueue(final String topic, final int queueId) throws ConsumeQueueException; /** * Get maximum offset of the topic queue. @@ -191,7 +193,7 @@ public interface MessageStore { * @param committed return the max offset in ConsumeQueue if true, or the max offset in CommitLog if false * @return Maximum offset at present. */ - long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed); + long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed) throws ConsumeQueueException; /** * Get the minimum offset of the topic queue. @@ -626,14 +628,6 @@ public interface MessageStore { void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, boolean isRecover, boolean isFileEnd) throws RocksDBException; - /** - * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) - * It will be triggered in two cases: - * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput - * @see CommitLog#recoverAbnormally - */ - void finishCommitLogDispatch(); - /** * Get the message store config * @@ -724,6 +718,7 @@ public interface MessageStore { * * @return the queue store */ + @Nonnull ConsumeQueueStoreInterface getQueueStore(); /** diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java deleted file mode 100644 index 5bc587a8e0..0000000000 --- a/store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.rocketmq.store; - -import org.apache.commons.lang3.StringUtils; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.MessageAccessor; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; - -/** - * MultiDispatch for lmq, not-thread-safe - */ -public class MultiDispatch { - private final StringBuilder keyBuilder = new StringBuilder(); - private final DefaultMessageStore messageStore; - private static final short VALUE_OF_EACH_INCREMENT = 1; - - public MultiDispatch(DefaultMessageStore messageStore) { - this.messageStore = messageStore; - } - - public String queueKey(String queueName, MessageExtBrokerInner msgInner) { - keyBuilder.delete(0, keyBuilder.length()); - keyBuilder.append(queueName); - keyBuilder.append('-'); - int queueId = msgInner.getQueueId(); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { - queueId = 0; - } - keyBuilder.append(queueId); - return keyBuilder.toString(); - } - - public void wrapMultiDispatch(final MessageExtBrokerInner msg) { - - String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - Long[] queueOffsets = new Long[queues.length]; - if (messageStore.getMessageStoreConfig().isEnableLmq()) { - for (int i = 0; i < queues.length; i++) { - String key = queueKey(queues[i], msg); - if (MixAll.isLmq(key)) { - queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(key); - } - } - } - MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, - StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); - msg.removeWaitStorePropertyString(); - } - - public void updateMultiQueueOffset(final MessageExtBrokerInner msgInner) { - String multiDispatchQueue = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); - String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); - for (String queue : queues) { - String key = queueKey(queue, msgInner); - if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { - messageStore.getQueueStore().increaseLmqOffset(key, VALUE_OF_EACH_INCREMENT); - } - } - } -} \ No newline at end of file diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java index 21f8d45c9d..0a7119cab1 100644 --- a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java @@ -79,15 +79,6 @@ public class RocksDBMessageStore extends DefaultMessageStore { this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); } - @Override - public void finishCommitLogDispatch() { - try { - putMessagePositionInfo(null); - } catch (RocksDBException e) { - ERROR_LOG.info("try to finish commitlog dispatch error.", e); - } - } - @Override public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { return findConsumeQueue(topic, queueId); diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java index 5195868e0f..8effe35bab 100644 --- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java @@ -428,8 +428,6 @@ public class MessageStoreConfig { private boolean rocksdbCQDoubleWriteEnable = false; - private int batchWriteKvCqSize = 16; - /** * If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type. * The following values are valid: @@ -447,14 +445,6 @@ public class MessageStoreConfig { */ private String bottomMostCompressionTypeForConsumeQueueStore = "zstd"; - public int getBatchWriteKvCqSize() { - return batchWriteKvCqSize; - } - - public void setBatchWriteKvCqSize(int batchWriteKvCqSize) { - this.batchWriteKvCqSize = batchWriteKvCqSize; - } - public boolean isRocksdbCQDoubleWriteEnable() { return rocksdbCQDoubleWriteEnable; } diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java new file mode 100644 index 0000000000..880e6347eb --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.exception; + +public class ConsumeQueueException extends StoreException { + public ConsumeQueueException() { + } + + public ConsumeQueueException(String message) { + super(message); + } + + public ConsumeQueueException(String message, Throwable cause) { + super(message, cause); + } + + public ConsumeQueueException(Throwable cause) { + super(cause); + } + + public ConsumeQueueException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java new file mode 100644 index 0000000000..8c99e8a05b --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.exception; + +public class StoreException extends Exception { + public StoreException() { + } + + public StoreException(String message) { + super(message); + } + + public StoreException(String message, Throwable cause) { + super(message, cause); + } + + public StoreException(Throwable cause) { + super(cause); + } + + public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java index 2401257c30..0f57a17d46 100644 --- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java @@ -46,6 +46,7 @@ import org.apache.rocketmq.store.StoreCheckpoint; import org.apache.rocketmq.store.StoreStatsService; import org.apache.rocketmq.store.TransientStorePool; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.hook.PutMessageHook; import org.apache.rocketmq.store.hook.SendMessageBackHook; @@ -63,7 +64,7 @@ import io.opentelemetry.sdk.metrics.InstrumentSelector; import io.opentelemetry.sdk.metrics.ViewBuilder; public abstract class AbstractPluginMessageStore implements MessageStore { - protected MessageStore next = null; + protected MessageStore next; protected MessageStorePluginContext context; public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) { @@ -139,12 +140,12 @@ public abstract class AbstractPluginMessageStore implements MessageStore { } @Override - public long getMaxOffsetInQueue(String topic, int queueId) { + public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException { return next.getMaxOffsetInQueue(topic, queueId); } @Override - public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) { + public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException { return next.getMaxOffsetInQueue(topic, queueId, committed); } @@ -647,11 +648,6 @@ public abstract class AbstractPluginMessageStore implements MessageStore { next.initMetrics(meter, attributesBuilderSupplier); } - @Override - public void finishCommitLogDispatch() { - next.finishCommitLogDispatch(); - } - @Override public void recoverTopicQueueTable() { next.recoverTopicQueueTable(); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java index d76b055773..dfce665d8f 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java @@ -25,6 +25,7 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.config.MessageStoreConfig; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.rocksdb.RocksDBException; public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { @@ -47,7 +48,7 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte } @Override - public Long getMaxOffset(String topic, int queueId) { + public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException { return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); } @@ -58,7 +59,7 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte } @Override - public ConcurrentMap getTopicQueueTable() { + public ConcurrentMap getTopicQueueTable() { return this.queueOffsetOperator.getTopicQueueTable(); } @@ -75,13 +76,13 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte } @Override - public void increaseLmqOffset(String queueKey, short messageNum) { - queueOffsetOperator.increaseLmqOffset(queueKey, messageNum); + public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException { + queueOffsetOperator.increaseLmqOffset(topic, queueId, delta); } @Override - public long getLmqQueueOffset(String queueKey) { - return queueOffsetOperator.getLmqOffset(queueKey); + public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException { + return queueOffsetOperator.getLmqOffset(topic, queueId, (t, q) -> 0L); } @Override @@ -105,9 +106,9 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte try { final long phyOffset = cqUnit.getPos(); final int size = cqUnit.getSize(); - long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); - return storeTime; + return this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); } catch (Exception e) { + log.error("Failed to getStoreTime", e); } } return -1; diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java index cbe9b4f5ac..5f9c2f90be 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java @@ -47,6 +47,7 @@ import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.StoreException; import static java.lang.String.format; import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; @@ -134,6 +135,12 @@ public class ConsumeQueueStore extends AbstractConsumeQueueStore { @Override public boolean shutdown() { + try { + flush(); + } catch (StoreException e) { + log.error("Failed to flush all consume queues", e); + return false; + } return true; } @@ -326,6 +333,15 @@ public class ConsumeQueueStore extends AbstractConsumeQueueStore { return fileQueueLifeCycle.flush(flushLeastPages); } + @Override + public void flush() throws StoreException { + for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { + for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { + flush(cqEntry.getValue(), 0); + } + } + } + @Override public void destroy(ConsumeQueueInterface consumeQueue) { FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java index e68880a828..72a481bd57 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java @@ -22,6 +22,8 @@ import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.common.BoundaryType; import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.rocksdb.RocksDBException; public interface ConsumeQueueStoreInterface { @@ -79,10 +81,17 @@ public interface ConsumeQueueStoreInterface { boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); /** - * clean expired data from minPhyOffset - * @param minPhyOffset + * Flush all nested consume queues to disk + * + * @throws StoreException if there is an error during flush */ - void cleanExpired(long minPhyOffset); + void flush() throws StoreException; + + /** + * clean expired data from minCommitLogOffset + * @param minCommitLogOffset Minimum commit log offset + */ + void cleanExpired(long minCommitLogOffset); /** * Check files. @@ -92,10 +101,10 @@ public interface ConsumeQueueStoreInterface { /** * Delete expired files ending at min commit log position. * @param consumeQueue - * @param minCommitLogPos min commit log position + * @param minCommitLogOffset min commit log position * @return deleted file numbers. */ - int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); + int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogOffset); /** * Is the first file available? @@ -185,17 +194,19 @@ public interface ConsumeQueueStoreInterface { /** * Increase lmq offset - * @param queueKey - * @param messageNum + * @param topic Topic/Queue name + * @param queueId Queue ID + * @param delta amount to increase */ - void increaseLmqOffset(String queueKey, short messageNum); + void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException; /** * get lmq queue offset - * @param queueKey + * @param topic + * @param queueId * @return */ - long getLmqQueueOffset(String queueKey); + long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException; /** * recover topicQueue table by minPhyOffset @@ -232,11 +243,13 @@ public interface ConsumeQueueStoreInterface { /** * get maxOffset of specific topic-queueId in topicQueue table - * @param topic - * @param queueId + * + * @param topic Topic name + * @param queueId Queue identifier * @return the max offset in QueueOffsetOperator + * @throws ConsumeQueueException if there is an error while retrieving max consume queue offset */ - Long getMaxOffset(String topic, int queueId); + Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException; /** * get max physic offset in consumeQueue diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java new file mode 100644 index 0000000000..a93ec0c50e --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue; + +import java.nio.charset.StandardCharsets; +import javax.annotation.Nonnull; +import org.apache.rocketmq.store.DispatchRequest; + +/** + * Use Record when Java 16 is available + */ +public class DispatchEntry { + public byte[] topic; + public int queueId; + public long queueOffset; + public long commitLogOffset; + public int messageSize; + public long tagCode; + public long storeTimestamp; + + public static DispatchEntry from(@Nonnull DispatchRequest request) { + DispatchEntry entry = new DispatchEntry(); + entry.topic = request.getTopic().getBytes(StandardCharsets.UTF_8); + entry.queueId = request.getQueueId(); + entry.queueOffset = request.getConsumeQueueOffset(); + entry.commitLogOffset = request.getCommitLogOffset(); + entry.messageSize = request.getMsgSize(); + entry.tagCode = request.getTagsCode(); + entry.storeTimestamp = request.getStoreTimestamp(); + return entry; + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java new file mode 100644 index 0000000000..8d3a8cd3a4 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.store.exception.ConsumeQueueException; + +public interface OffsetInitializer { + long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException; +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java new file mode 100644 index 0000000000..4b889e1e44 --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.store.queue; + +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.rocksdb.RocksDBException; +import org.apache.rocketmq.logging.org.slf4j.Logger; +import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + +public class OffsetInitializerRocksDBImpl implements OffsetInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(OffsetInitializerRocksDBImpl.class); + + private final RocksDBConsumeQueueStore consumeQueueStore; + + public OffsetInitializerRocksDBImpl(RocksDBConsumeQueueStore consumeQueueStore) { + this.consumeQueueStore = consumeQueueStore; + } + + @Override + public long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException { + try { + long offset = consumeQueueStore.getMaxOffsetInQueue(topic, queueId); + LOGGER.info("Look up RocksDB for max-offset of LMQ[{}:{}]: {}", topic, queueId, offset); + return offset; + } catch (RocksDBException e) { + throw new ConsumeQueueException(e); + } + } +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java index 5b4bf994e0..7d38817161 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.queue; +import com.google.common.base.Preconditions; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -26,6 +27,7 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.store.exception.ConsumeQueueException; /** * QueueOffsetOperator is a component for operating offsets for queues. @@ -35,7 +37,11 @@ public class QueueOffsetOperator { private ConcurrentMap topicQueueTable = new ConcurrentHashMap<>(1024); private ConcurrentMap batchTopicQueueTable = new ConcurrentHashMap<>(1024); - private ConcurrentMap lmqTopicQueueTable = new ConcurrentHashMap<>(1024); + + /** + * {TOPIC}-{QUEUE_ID} --> NEXT Consume Queue Offset + */ + private ConcurrentMap lmqTopicQueueTable = new ConcurrentHashMap<>(1024); public long getQueueOffset(String topicQueueKey) { return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); @@ -63,17 +69,28 @@ public class QueueOffsetOperator { this.batchTopicQueueTable.put(topicQueueKey, batchQueueOffset + messageNum); } - public long getLmqOffset(String topicQueueKey) { - return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); + public long getLmqOffset(String topic, int queueId, OffsetInitializer callback) throws ConsumeQueueException { + Preconditions.checkNotNull(callback, "ConsumeQueueOffsetCallback cannot be null"); + String topicQueue = topic + "-" + queueId; + if (!lmqTopicQueueTable.containsKey(topicQueue)) { + // Load from RocksDB on cache miss. + Long prev = lmqTopicQueueTable.putIfAbsent(topicQueue, callback.maxConsumeQueueOffset(topic, queueId)); + if (null != prev) { + log.error("[BUG] Data racing, lmqTopicQueueTable should NOT contain key={}", topicQueue); + } + } + return lmqTopicQueueTable.get(topicQueue); } - public Long getLmqTopicQueueNextOffset(String topicQueueKey) { - return this.lmqTopicQueueTable.get(topicQueueKey); - } - - public void increaseLmqOffset(String queueKey, short messageNum) { - Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, queueKey, k -> 0L); - this.lmqTopicQueueTable.put(queueKey, lmqOffset + messageNum); + public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException { + String topicQueue = topic + "-" + queueId; + if (!this.lmqTopicQueueTable.containsKey(topicQueue)) { + throw new ConsumeQueueException(String.format("Max offset of Queue[name=%s, id=%d] should have existed", topic, queueId)); + } + long prev = lmqTopicQueueTable.get(topicQueue); + this.lmqTopicQueueTable.compute(topicQueue, (k, offset) -> offset + delta); + long current = lmqTopicQueueTable.get(topicQueue); + log.debug("Max offset of LMQ[{}:{}] increased: {} --> {}", topic, queueId, prev, current); } public long currentQueueOffset(String topicQueueKey) { @@ -112,4 +129,4 @@ public class QueueOffsetOperator { public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { this.batchTopicQueueTable = batchTopicQueueTable; } -} \ No newline at end of file +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java index 6fa66282e9..889131d1cc 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java @@ -16,7 +16,10 @@ */ package org.apache.rocketmq.store.queue; +import io.netty.util.internal.PlatformDependent; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -24,31 +27,33 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; -import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.ConsumeQueue; import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.DispatchRequest; +import org.apache.rocketmq.store.queue.offset.OffsetEntry; +import org.apache.rocketmq.store.queue.offset.OffsetEntryType; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.WriteBatch; -import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1; public class RocksDBConsumeQueueOffsetTable { private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); - private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); + private static final byte[] MAX_BYTES = "max".getBytes(StandardCharsets.UTF_8); + private static final byte[] MIN_BYTES = "min".getBytes(StandardCharsets.UTF_8); /** * Rocksdb ConsumeQueue's Offset unit. Format: @@ -72,10 +77,9 @@ public class RocksDBConsumeQueueOffsetTable { * * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes */ - private static final int OFFSET_PHY_OFFSET = 0; - private static final int OFFSET_CQ_OFFSET = 8; + static final int OFFSET_PHY_OFFSET = 0; + static final int OFFSET_CQ_OFFSET = 8; /** - * * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ @@ -86,16 +90,18 @@ public class RocksDBConsumeQueueOffsetTable { /** * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. + * * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of * RocksDBConsumeQueueOffsetTable to find it. */ private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; - private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); + private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(StandardCharsets.UTF_8); private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; private final ByteBuffer maxPhyOffsetBB; + static { buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); @@ -111,69 +117,147 @@ public class RocksDBConsumeQueueOffsetTable { /** * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them * from heap to avoid accessing rocksdb. + * * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset */ - private final Map topicQueueMinOffset; - private final Map topicQueueMaxCqOffset; + private final ConcurrentMap topicQueueMinOffset; + private final ConcurrentMap topicQueueMaxCqOffset; public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; this.rocksDBStorage = rocksDBStorage; this.messageStore = messageStore; - this.topicQueueMinOffset = new ConcurrentHashMap(1024); - this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); + this.topicQueueMinOffset = new ConcurrentHashMap<>(1024); + this.topicQueueMaxCqOffset = new ConcurrentHashMap<>(1024); this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); } public void load() { this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); + loadMaxConsumeQueueOffsets(); } - public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, - final byte[] topicBytes, final DispatchRequest request, - final Map> tempTopicQueueMaxOffsetMap) { - buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); - ByteBuffer topicQueueId = offsetBBPair.getObject1(); - ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); - Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); - if (old == null) { - tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); - } else { - long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); - long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); - if (maxOffset >= oldMaxOffset) { - ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); - } + private void loadMaxConsumeQueueOffsets() { + Function predicate = entry -> entry.type == OffsetEntryType.MAXIMUM; + Consumer fn = entry -> { + topicQueueMaxCqOffset.putIfAbsent(entry.topic + "-" + entry.queueId, entry.offset); + ROCKSDB_LOG.info("Max {}:{} --> {}|{}", entry.topic, entry.queueId, entry.offset, entry.commitLogOffset); + }; + try { + forEach(predicate, fn); + } catch (RocksDBException e) { + log.error("Failed to maximum consume queue offset", e); } } - public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, + public void forEach(Function predicate, Consumer fn) throws RocksDBException { + try (RocksIterator iterator = this.rocksDBStorage.seekOffsetCF()) { + if (null == iterator) { + return; + } + + int keyBufferCapacity = 256; + iterator.seekToFirst(); + ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity); + ByteBuffer valueBuffer = ByteBuffer.allocateDirect(16); + while (iterator.isValid()) { + // parse key buffer according to key layout + keyBuffer.clear(); // clear position and limit before reuse + int total = iterator.key(keyBuffer); + if (total > keyBufferCapacity) { + keyBufferCapacity = total; + PlatformDependent.freeDirectBuffer(keyBuffer); + keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity); + continue; + } + + if (keyBuffer.remaining() <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES) { + iterator.next(); + ROCKSDB_LOG.warn("Malformed Key/Value pair"); + continue; + } + + int topicLength = keyBuffer.getInt(); + byte ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + + byte[] topicBytes = new byte[topicLength]; + keyBuffer.get(topicBytes); + ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + String topic = new String(topicBytes, StandardCharsets.UTF_8); + + byte[] minMax = new byte[3]; + keyBuffer.get(minMax); + OffsetEntryType entryType; + if (Arrays.equals(minMax, MAX_BYTES)) { + entryType = OffsetEntryType.MAXIMUM; + } else { + entryType = OffsetEntryType.MINIMUM; + } + ctrl1 = keyBuffer.get(); + assert ctrl1 == CTRL_1; + + assert keyBuffer.remaining() == Integer.BYTES; + int queueId = keyBuffer.getInt(); + + // Read and parse value buffer according to value layout + valueBuffer.clear(); // clear position and limit before reuse + total = iterator.value(valueBuffer); + if (total != Long.BYTES + Long.BYTES) { + // Skip system checkpoint topic as its value is only 8 bytes + iterator.next(); + continue; + } + long commitLogOffset = valueBuffer.getLong(); + long consumeOffset = valueBuffer.getLong(); + + OffsetEntry entry = new OffsetEntry(); + entry.topic = topic; + entry.queueId = queueId; + entry.type = entryType; + entry.offset = consumeOffset; + entry.commitLogOffset = commitLogOffset; + if (predicate.apply(entry)) { + fn.accept(entry); + } + iterator.next(); + } + // clean up direct buffers + PlatformDependent.freeDirectBuffer(keyBuffer); + PlatformDependent.freeDirectBuffer(valueBuffer); + } + } + + public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { - for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); } appendMaxPhyOffset(writeBatch, maxPhyOffset); } - public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { - for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { - DispatchRequest request = entry.getValue().getObject2(); - putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); + public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { + for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { + DispatchEntry dispatchEntry = entry.getValue().getObject2(); + String topic = new String(dispatchEntry.topic, StandardCharsets.UTF_8); + putHeapMaxCqOffset(topic, dispatchEntry.queueId, dispatchEntry.queueOffset); } } /** * When topic is deleted, we clean up its offset info in rocksdb. + * * @param topic * @param queueId * @throws RocksDBException */ public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; @@ -214,15 +298,14 @@ public class RocksDBConsumeQueueOffsetTable { /** * Traverse the offset table to find dirty topic + * * @param existTopicSet * @return */ public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { Map> topicQueueIdToBeDeletedMap = new HashMap<>(); - RocksIterator iterator = null; - try { - iterator = rocksDBStorage.seekOffsetCF(); + try (RocksIterator iterator = rocksDBStorage.seekOffsetCF()) { if (iterator == null) { return topicQueueIdToBeDeletedMap; } @@ -236,17 +319,22 @@ public class RocksDBConsumeQueueOffsetTable { ByteBuffer keyBB = ByteBuffer.wrap(key); int topicLen = keyBB.getInt(0); byte[] topicBytes = new byte[topicLen]; - /** + /* * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 */ keyBB.position(4 + 1); keyBB.get(topicBytes); - String topic = new String(topicBytes, CHARSET_UTF8); + String topic = new String(topicBytes, StandardCharsets.UTF_8); if (TopicValidator.isSystemTopic(topic)) { continue; } - /** + // LMQ topic offsets should NOT be removed + if (MixAll.isLmq(topic)) { + continue; + } + + /* * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" * = 4 + 1 + topicLen + 1 + 3 + 1 */ @@ -270,10 +358,6 @@ public class RocksDBConsumeQueueOffsetTable { } } catch (Exception e) { ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); - } finally { - if (iterator != null) { - iterator.close(); - } } return topicQueueIdToBeDeletedMap; } @@ -285,9 +369,13 @@ public class RocksDBConsumeQueueOffsetTable { final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; String topicQueueId = buildTopicQueueId(topic, queueId); - this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); + long offset = maxCqOffset != null ? maxCqOffset : -1L; + Long prev = this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, offset); + if (null == prev) { + ROCKSDB_LOG.info("Max offset of {} is initialized to {} according to RocksDB", topicQueueId, offset); + } if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { - ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); + ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, offset); } } @@ -296,34 +384,50 @@ public class RocksDBConsumeQueueOffsetTable { /** * truncate dirty offset in rocksdb + * * @param offsetToTruncate * @throws RocksDBException */ public void truncateDirty(long offsetToTruncate) throws RocksDBException { correctMaxPyhOffset(offsetToTruncate); - ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); - if (allTopicConfigMap == null) { - return; - } - for (TopicConfig topicConfig : allTopicConfigMap.values()) { - for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { - truncateDirtyOffset(topicConfig.getTopicName(), i); + Function predicate = entry -> { + if (entry.type == OffsetEntryType.MINIMUM) { + return false; } - } + // Normal entry offset MUST have the following inequality + // entry commit-log offset + message-size-in-bytes <= offsetToTruncate; + // otherwise, the consume queue contains dirty records to truncate; + // + // If the broker node is configured to use async-flush, it's possible consume queues contain + // pointers to message records that is not flushed and lost during restart. + return entry.commitLogOffset >= offsetToTruncate; + }; + + Consumer fn = entry -> { + try { + truncateDirtyOffset(entry.topic, entry.queueId); + } catch (RocksDBException e) { + log.error("Failed to truncate maximum offset of consume queue[topic={}, queue-id={}]", + entry.topic, entry.queueId, e); + } + }; + + forEach(predicate, fn); } - private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { + private Pair isMinOffsetOk(final String topic, final int queueId, + final long minPhyOffset) throws RocksDBException { PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); if (phyAndCQOffset != null) { final long phyOffset = phyAndCQOffset.getPhyOffset(); final long cqOffset = phyAndCQOffset.getCqOffset(); - return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); + return (phyOffset >= minPhyOffset) ? new Pair<>(true, cqOffset) : new Pair<>(false, cqOffset); } ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); if (byteBuffer == null) { - return new Pair(false, 0L); + return new Pair<>(false, 0L); } final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); @@ -334,9 +438,9 @@ public class RocksDBConsumeQueueOffsetTable { if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); } - return new Pair(true, cqOffset); + return new Pair<>(true, cqOffset); } - return new Pair(false, cqOffset); + return new Pair<>(false, cqOffset); } private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { @@ -361,8 +465,7 @@ public class RocksDBConsumeQueueOffsetTable { if (!this.rocksDBStorage.hold()) { return; } - try { - WriteBatch writeBatch = new WriteBatch(); + try (WriteBatch writeBatch = new WriteBatch()) { long oldMaxPhyOffset = getMaxPhyOffset(); if (oldMaxPhyOffset <= maxPhyOffset) { return; @@ -416,10 +519,10 @@ public class RocksDBConsumeQueueOffsetTable { } private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); - byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); + byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); return (value != null) ? ByteBuffer.wrap(value) : null; } @@ -427,17 +530,19 @@ public class RocksDBConsumeQueueOffsetTable { return topic + "-" + queueId; } - private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { + private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, + final long minCQOffset) { String topicQueueId = buildTopicQueueId(topic, queueId); PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); } - private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { + private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxOffset) { String topicQueueId = buildTopicQueueId(topic, queueId); - Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); - if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { - ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); + Long prev = this.topicQueueMaxCqOffset.put(topicQueueId, maxOffset); + if (prev != null && prev > maxOffset) { + ERROR_LOG.error("Max offset of consume-queue[topic={}, queue-id={}] regressed. prev-max={}, current-max={}", + topic, queueId, prev, maxOffset); } } @@ -463,9 +568,8 @@ public class RocksDBConsumeQueueOffsetTable { if (!this.rocksDBStorage.hold()) { return; } - WriteBatch writeBatch = new WriteBatch(); - try { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + try (WriteBatch writeBatch = new WriteBatch()) { + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); @@ -481,7 +585,6 @@ public class RocksDBConsumeQueueOffsetTable { ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); throw e; } finally { - writeBatch.close(); this.rocksDBStorage.release(); if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", @@ -504,10 +607,8 @@ public class RocksDBConsumeQueueOffsetTable { throw new RocksDBException("correctMaxCqOffset error"); } - long high = maxCQOffset; - long low = minCQOffset; - PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, - low, maxPhyOffsetInCQ, false); + PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset, + minCQOffset, maxPhyOffsetInCQ, false); long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); @@ -541,10 +642,8 @@ public class RocksDBConsumeQueueOffsetTable { return true; } - long high = maxCQOffset; - long low = minCQOffset; - PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, - minPhyOffset, true); + PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset, + minCQOffset, minPhyOffset, true); long targetCQOffset = phyAndCQOffset.getCqOffset(); long targetPhyOffset = phyAndCQOffset.getPhyOffset(); @@ -568,28 +667,29 @@ public class RocksDBConsumeQueueOffsetTable { return new Pair<>(offsetKey, offsetValue); } - private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, - final byte[] topicBytes, final DispatchRequest request) { + static void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, + final DispatchEntry entry) { final ByteBuffer offsetKey = offsetBBPair.getObject1(); - buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); + buildOffsetKeyByteBuffer(offsetKey, entry.topic, entry.queueId, true); final ByteBuffer offsetValue = offsetBBPair.getObject2(); - buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); + buildOffsetValueByteBuffer(offsetValue, entry.commitLogOffset, entry.queueOffset); } - private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { + private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); return byteBuffer; } - private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { + private static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, + final int queueId, final boolean max) { byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); } - private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, - final boolean max) { + private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, + final int queueId, final boolean max) { byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); if (max) { byteBuffer.put(MAX_BYTES); @@ -600,18 +700,20 @@ public class RocksDBConsumeQueueOffsetTable { byteBuffer.flip(); } - private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + private static void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, + final long cqOffset) { byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); } - private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { + private static ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); return byteBuffer; } - private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { + private static void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, + final long cqOffset) { byteBuffer.putLong(phyOffset).putLong(cqOffset); byteBuffer.flip(); } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java index 17b845d817..67a0015743 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java @@ -18,6 +18,7 @@ package org.apache.rocketmq.store.queue; import java.io.File; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -28,21 +29,27 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + +import javax.annotation.Nonnull; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.common.BoundaryType; +import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.Pair; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.common.utils.DataConverter; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; +import org.apache.rocketmq.store.exception.StoreException; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; +import org.rocksdb.FlushOptions; import org.rocksdb.RocksDBException; import org.rocksdb.Statistics; import org.rocksdb.WriteBatch; @@ -51,11 +58,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); - public static final byte CTRL_0 = '\u0000'; - public static final byte CTRL_1 = '\u0001'; - public static final byte CTRL_2 = '\u0002'; + private static final int DEFAULT_BYTE_BUFFER_CAPACITY = 16; - private final int batchSize; public static final int MAX_KEY_LEN = 300; private final ScheduledExecutorService scheduledExecutorService; @@ -70,13 +74,16 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; - private final WriteBatch writeBatch; - private final List bufferDRList; private final List> cqBBPairList; private final List> offsetBBPairList; - private final Map> tempTopicQueueMaxOffsetMap; + private final Map> tempTopicQueueMaxOffsetMap; private volatile boolean isCQError = false; + private int consumeQueueByteBufferCacheIndex; + private int offsetBufferCacheIndex; + + private final OffsetInitializer offsetInitializer; + public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { super(messageStore); @@ -85,12 +92,10 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); - this.writeBatch = new WriteBatch(); - this.batchSize = messageStoreConfig.getBatchWriteKvCqSize(); - this.bufferDRList = new ArrayList<>(batchSize); - this.cqBBPairList = new ArrayList<>(batchSize); - this.offsetBBPairList = new ArrayList<>(batchSize); - for (int i = 0; i < batchSize; i++) { + this.offsetInitializer = new OffsetInitializerRocksDBImpl(this); + this.cqBBPairList = new ArrayList<>(16); + this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY); + for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) { this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); } @@ -100,6 +105,22 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); } + private Pair getCQByteBufferPair() { + int idx = consumeQueueByteBufferCacheIndex++; + if (idx >= cqBBPairList.size()) { + this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); + } + return cqBBPairList.get(idx); + } + + private Pair getOffsetByteBufferPair() { + int idx = offsetBufferCacheIndex++; + if (idx >= offsetBBPairList.size()) { + this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); + } + return offsetBBPairList.get(idx); + } + @Override public void start() { log.info("RocksDB ConsumeQueueStore start!"); @@ -164,19 +185,19 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { @Override public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { - if (request == null || this.bufferDRList.size() >= batchSize) { - putMessagePosition(); - } - - if (request != null) { - this.bufferDRList.add(request); + if (null == request) { + return; } + // We are taking advantage of Atomic Flush, this operation is purely memory-based. + // batch and cache in Java heap does not make sense, instead, we should put the metadata into RocksDB immediately + // to optimized overall end-to-end latency. + putMessagePosition(request); } - public void putMessagePosition() throws RocksDBException { + public void putMessagePosition(DispatchRequest request) throws RocksDBException { final int maxRetries = 30; for (int i = 0; i < maxRetries; i++) { - if (putMessagePosition0()) { + if (putMessagePosition0(request)) { if (this.isCQError) { this.messageStore.getRunningFlags().clearLogicsQueueError(); this.isCQError = false; @@ -198,81 +219,113 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { throw new RocksDBException("put CQ Failed"); } - private boolean putMessagePosition0() { + private boolean putMessagePosition0(DispatchRequest request) { if (!this.rocksDBStorage.hold()) { return false; } - final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; - try { - final List bufferDRList = this.bufferDRList; - final int size = bufferDRList.size(); - if (size == 0) { - return true; - } - final List> cqBBPairList = this.cqBBPairList; - final List> offsetBBPairList = this.offsetBBPairList; - final WriteBatch writeBatch = this.writeBatch; - + try (WriteBatch writeBatch = new WriteBatch()) { long maxPhyOffset = 0; - for (int i = size - 1; i >= 0; i--) { - final DispatchRequest request = bufferDRList.get(i); - final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); + DispatchEntry entry = DispatchEntry.from(request); + dispatch(entry, writeBatch); + dispatchLMQ(request, writeBatch); - this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); - this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), - topicBytes, request, tempTopicQueueMaxOffsetMap); - - final int msgSize = request.getMsgSize(); - final long phyOffset = request.getCommitLogOffset(); - if (phyOffset + msgSize >= maxPhyOffset) { - maxPhyOffset = phyOffset + msgSize; - } + final int msgSize = request.getMsgSize(); + final long phyOffset = request.getCommitLogOffset(); + if (phyOffset + msgSize >= maxPhyOffset) { + maxPhyOffset = phyOffset + msgSize; } this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); - // clear writeBatch in batchPut this.rocksDBStorage.batchPut(writeBatch); - this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); - - long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); + long storeTimeStamp = request.getStoreTimestamp(); if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); } this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); - - notifyMessageArriveAndClear(); + notifyMessageArrival(request); return true; } catch (Exception e) { - ERROR_LOG.error("putMessagePosition0 Failed.", e); + ERROR_LOG.error("putMessagePosition0 failed.", e); return false; } finally { tempTopicQueueMaxOffsetMap.clear(); + consumeQueueByteBufferCacheIndex = 0; + offsetBufferCacheIndex = 0; this.rocksDBStorage.release(); } } - private void notifyMessageArriveAndClear() { - final List bufferDRList = this.bufferDRList; - try { - for (DispatchRequest dp : bufferDRList) { - this.messageStore.notifyMessageArriveIfNecessary(dp); + private void dispatch(@Nonnull DispatchEntry entry, @Nonnull final WriteBatch writeBatch) throws RocksDBException { + this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(getCQByteBufferPair(), entry, writeBatch); + updateTempTopicQueueMaxOffset(getOffsetByteBufferPair(), entry); + } + + private void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, + final DispatchEntry entry) { + RocksDBConsumeQueueOffsetTable.buildOffsetKeyAndValueByteBuffer(offsetBBPair, entry); + ByteBuffer topicQueueId = offsetBBPair.getObject1(); + ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); + Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); + if (old == null) { + tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair<>(maxOffsetBB, entry)); + } else { + long oldMaxOffset = old.getObject1().getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET); + long maxOffset = maxOffsetBB.getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET); + if (maxOffset >= oldMaxOffset) { + ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); } + } + } + + private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteBatch writeBatch) + throws RocksDBException { + if (!messageStoreConfig.isEnableLmq() || !request.containsLMQ()) { + return; + } + Map map = request.getPropertiesMap(); + String lmqNames = map.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); + String lmqOffsets = map.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); + String[] queues = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR); + String[] queueOffsets = lmqOffsets.split(MixAll.LMQ_DISPATCH_SEPARATOR); + if (queues.length != queueOffsets.length) { + ERROR_LOG.error("[bug] queues.length!=queueOffsets.length ", request.getTopic()); + return; + } + for (int i = 0; i < queues.length; i++) { + String queueName = queues[i]; + DispatchEntry entry = DispatchEntry.from(request); + long queueOffset = Long.parseLong(queueOffsets[i]); + int queueId = request.getQueueId(); + if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { + queueId = MixAll.LMQ_QUEUE_ID; + } + entry.queueId = queueId; + entry.queueOffset = queueOffset; + entry.topic = queueName.getBytes(StandardCharsets.UTF_8); + log.debug("Dispatch LMQ[{}:{}]:{} --> {}", queueName, queueId, queueOffset, entry.commitLogOffset); + dispatch(entry, writeBatch); + } + } + + private void notifyMessageArrival(DispatchRequest request) { + try { + this.messageStore.notifyMessageArriveIfNecessary(request); } catch (Exception e) { ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); - } finally { - bufferDRList.clear(); } } public Statistics getStatistics() { return rocksDBStorage.getStatistics(); } + @Override - public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { + public List rangeQuery(final String topic, final int queueId, final long startIndex, + final int num) throws RocksDBException { return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); } @@ -284,6 +337,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { /** * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them * when we use them, we call it lazy correction. + * * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) */ @@ -310,8 +364,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { return; } - WriteBatch writeBatch = new WriteBatch(); - try { + try (WriteBatch writeBatch = new WriteBatch()) { this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); @@ -320,7 +373,6 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); throw e; } finally { - writeBatch.close(); this.rocksDBStorage.release(); } } @@ -330,10 +382,22 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { try { this.rocksDBStorage.flushWAL(); } catch (Exception e) { + log.error("Failed to flush WAL", e); } return true; } + @Override + public void flush() throws StoreException { + try (FlushOptions flushOptions = new FlushOptions()) { + flushOptions.setWaitForFlush(true); + flushOptions.setAllowWriteStall(true); + this.rocksDBStorage.flush(flushOptions); + } catch (RocksDBException e) { + throw new StoreException(e); + } + } + @Override public void checkSelf() { // ignored @@ -350,8 +414,9 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). - * @param offsetToTruncate - * @throws RocksDBException + * + * @param offsetToTruncate CommitLog offset to truncate to + * @throws RocksDBException If there is any error. */ @Override public void truncateDirty(long offsetToTruncate) throws RocksDBException { @@ -369,7 +434,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { } @Override - public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, + BoundaryType boundaryType) throws RocksDBException { final long minPhysicOffset = this.messageStore.getMinPhyOffset(); long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); @@ -380,6 +446,15 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { minPhysicOffset, boundaryType); } + /** + * This method actually returns NEXT slot index to use, starting from 0. For example, if the queue is empty, + * it returns 0, pointing to the first slot of the 0-based queue; + * + * @param topic Topic name + * @param queueId Queue ID + * @return Index of the next slot to push into + * @throws RocksDBException if RocksDB fails to fulfill the request. + */ @Override public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); @@ -444,4 +519,17 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { public long getTotalSize() { return 0; } + + @Override + public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException { + return queueOffsetOperator.getLmqOffset(topic, queueId, offsetInitializer); + } + + @Override + public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException { + if (MixAll.isLmq(topic)) { + return getLmqQueueOffset(topic, queueId); + } + return super.getMaxOffset(topic, queueId); + } } diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java index 194bd4cca5..deee029570 100644 --- a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.store.queue; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -26,17 +27,15 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; import org.apache.rocketmq.store.DefaultMessageStore; -import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.RocksDBException; import org.rocksdb.WriteBatch; -import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; -import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_0; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1; +import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_2; /** * We use RocksDBConsumeQueueTable to store cqUnit. @@ -105,30 +104,30 @@ public class RocksDBConsumeQueueTable { this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); } - public void buildAndPutCQByteBuffer(final Pair cqBBPair, - final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { + public void buildAndPutCQByteBuffer(final Pair cqBBPair, final DispatchEntry request, + final WriteBatch writeBatch) throws RocksDBException { final ByteBuffer cqKey = cqBBPair.getObject1(); - buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); + buildCQKeyByteBuffer(cqKey, request.topic, request.queueId, request.queueOffset); final ByteBuffer cqValue = cqBBPair.getObject2(); - buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); + buildCQValueByteBuffer(cqValue, request.commitLogOffset, request.messageSize, request.tagCode, request.storeTimestamp); writeBatch.put(this.defaultCFH, cqKey, cqValue); } public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); return (value != null) ? ByteBuffer.wrap(value) : null; } public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); - final List defaultCFHList = new ArrayList(num); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); + final List defaultCFHList = new ArrayList<>(num); final ByteBuffer[] resultList = new ByteBuffer[num]; - final List kvIndexList = new ArrayList(num); - final List kvKeyList = new ArrayList(num); + final List kvIndexList = new ArrayList<>(num); + final List kvKeyList = new ArrayList<>(num); for (int i = 0; i < num; i++) { final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); kvIndexList.add(i); @@ -153,7 +152,7 @@ public class RocksDBConsumeQueueTable { } final int resultSize = resultList.length; - List bbValueList = new ArrayList(resultSize); + List bbValueList = new ArrayList<>(resultSize); for (int i = 0; i < resultSize; i++) { ByteBuffer byteBuffer = resultList[i]; if (byteBuffer == null) { @@ -171,7 +170,7 @@ public class RocksDBConsumeQueueTable { * @throws RocksDBException */ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { - final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); + final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8); final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java new file mode 100644 index 0000000000..bac3aa430b --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue.offset; + +public class OffsetEntry { + /** + * Topic identifier. For now, it's topic name directly. In the future, we should use fixed length topic identifier. + */ + public String topic; + + /** + * Queue ID + */ + public int queueId; + + /** + * Flag if the entry is for maximum or minimum + */ + public OffsetEntryType type; + + /** + * Maximum or minimum consume-queue offset. + */ + public long offset; + + /** + * Maximum or minimum commit-log offset. + */ + public long commitLogOffset; +} diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java new file mode 100644 index 0000000000..78df4e15fa --- /dev/null +++ b/store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.store.queue.offset; + +public enum OffsetEntryType { + MAXIMUM, + MINIMUM +} diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java index 1d09ca86ec..eee38e0a8f 100644 --- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java @@ -54,6 +54,7 @@ import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -374,7 +375,7 @@ public class DefaultMessageStoreTest { } @Test - public void testPutMessage_whenMessagePropertyIsTooLong() { + public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException { String topicName = "messagePropertyIsTooLongTest"; MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); @@ -539,7 +540,7 @@ public class DefaultMessageStoreTest { } @Test - public void testMaxOffset() throws InterruptedException { + public void testMaxOffset() throws InterruptedException, ConsumeQueueException { int firstBatchMessages = 3; int queueId = 0; messageBody = storeMessage.getBytes(); diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java deleted file mode 100644 index eae5eaa07a..0000000000 --- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.rocketmq.store; - -import java.io.File; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; - -import java.util.concurrent.ConcurrentHashMap; -import org.apache.rocketmq.common.BrokerConfig; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageDecoder; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; -import org.apache.rocketmq.store.config.MessageStoreConfig; -import org.apache.rocketmq.store.queue.MultiDispatchUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.rocksdb.RocksDBException; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class MultiDispatchTest { - - private MultiDispatch multiDispatch; - - private DefaultMessageStore messageStore; - - @Before - public void init() throws Exception { - MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); - messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); - messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); - messageStoreConfig.setMaxHashSlotNum(100); - messageStoreConfig.setMaxIndexNum(100 * 10); - messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1"); - messageStoreConfig.setStorePathCommitLog( - System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1" + File.separator + "commitlog"); - - messageStoreConfig.setEnableLmq(true); - messageStoreConfig.setEnableMultiDispatch(true); - BrokerConfig brokerConfig = new BrokerConfig(); - //too much reference - messageStore = new DefaultMessageStore(messageStoreConfig, null, null, brokerConfig, new ConcurrentHashMap<>()); - multiDispatch = new MultiDispatch(messageStore); - } - - @After - public void destroy() { - UtilAll.deleteFile(new File(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1")); - } - - @Test - public void lmqQueueKey() { - MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); - when(messageExtBrokerInner.getQueueId()).thenReturn(2); - String ret = MultiDispatchUtils.lmqQueueKey("%LMQ%lmq123"); - assertEquals(ret, "%LMQ%lmq123-0"); - } - - @Test - public void wrapMultiDispatch() throws RocksDBException { - MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); - multiDispatch.wrapMultiDispatch(messageExtBrokerInner); - assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); - } - - private MessageExtBrokerInner buildMessageMultiQueue() { - MessageExtBrokerInner msg = new MessageExtBrokerInner(); - msg.setTopic("test"); - msg.setTags("TAG1"); - msg.setKeys("Hello"); - msg.setBody("aaa".getBytes(Charset.forName("UTF-8"))); - msg.setKeys(String.valueOf(System.currentTimeMillis())); - msg.setQueueId(0); - msg.setSysFlag(0); - msg.setBornTimestamp(System.currentTimeMillis()); - msg.setStoreHost(new InetSocketAddress("127.0.0.1", 54270)); - msg.setBornHost(new InetSocketAddress("127.0.0.1", 10911)); - for (int i = 0; i < 1; i++) { - msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, "%LMQ%123,%LMQ%456"); - } - msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); - - return msg; - } -} \ No newline at end of file diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java index 2af07197a5..f934f80364 100644 --- a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java @@ -56,6 +56,7 @@ import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.FlushDiskType; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -434,7 +435,7 @@ public class RocksDBMessageStoreTest { } @Test - public void testPutMessage_whenMessagePropertyIsTooLong() { + public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException { if (notExecuted()) { return; } @@ -603,7 +604,7 @@ public class RocksDBMessageStoreTest { } @Test - public void testMaxOffset() { + public void testMaxOffset() throws ConsumeQueueException { if (notExecuted()) { return; } diff --git a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java index ad521440e9..bfed96e8cd 100644 --- a/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java @@ -36,6 +36,7 @@ import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; import org.apache.rocketmq.remoting.protocol.filter.FilterAPI; import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.store.DefaultMessageFilter; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.test.base.BaseConf; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -80,7 +81,7 @@ public class LagCalculationIT extends BaseConf { shutdown(); } - private Pair getLag(List mqs) { + private Pair getLag(List mqs) throws ConsumeQueueException { long lag = 0; long pullLag = 0; for (BrokerController controller : brokerControllerList) { @@ -120,7 +121,7 @@ public class LagCalculationIT extends BaseConf { } @Test - public void testCalculateLag() { + public void testCalculateLag() throws ConsumeQueueException { int msgSize = 10; List mqs = producer.getMessageQueue(); MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize); diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java index ee06700b8b..982909c5ee 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java @@ -34,6 +34,7 @@ import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.store.DispatchRequest; import org.apache.rocketmq.store.MessageStore; import org.apache.rocketmq.store.SelectMappedBufferResult; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.store.queue.ConsumeQueueInterface; import org.apache.rocketmq.store.queue.CqUnit; import org.apache.rocketmq.tieredstore.MessageStoreConfig; @@ -259,6 +260,10 @@ public class MessageStoreDispatcherImpl extends ServiceThread implements Message } ); } + } catch (ConsumeQueueException e) { + CompletableFuture future = new CompletableFuture<>(); + future.completeExceptionally(e); + return future; } finally { flatFile.getFileLock().unlock(); } diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java index 8b5a9e63c0..4d08328483 100644 --- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java @@ -40,6 +40,7 @@ import org.apache.rocketmq.common.metrics.NopLongCounter; import org.apache.rocketmq.common.metrics.NopLongHistogram; import org.apache.rocketmq.common.metrics.NopObservableLongGauge; import org.apache.rocketmq.store.MessageStore; +import org.apache.rocketmq.store.exception.ConsumeQueueException; import org.apache.rocketmq.tieredstore.MessageStoreConfig; import org.apache.rocketmq.tieredstore.common.FileSegmentType; import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher; @@ -177,26 +178,30 @@ public class TieredStoreMetricsManager { .ofLongs() .buildWithCallback(measurement -> { for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) { + try { - MessageQueue mq = flatFile.getMessageQueue(); - long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); - long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { - continue; + MessageQueue mq = flatFile.getMessageQueue(); + long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); + long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { + continue; + } + + Attributes commitLogAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) + .build(); + + Attributes consumeQueueAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) + .build(); + measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes); + } catch (ConsumeQueueException e) { + // TODO: handle exception here } - - Attributes commitLogAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) - .build(); - - Attributes consumeQueueAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) - .build(); - measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes); } }); @@ -206,31 +211,35 @@ public class TieredStoreMetricsManager { .ofLongs() .buildWithCallback(measurement -> { for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) { + try { + MessageQueue mq = flatFile.getMessageQueue(); - MessageQueue mq = flatFile.getMessageQueue(); - long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); - long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); - if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { - continue; - } + long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId()); + long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1); + if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) { + continue; + } - Attributes commitLogAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) - .build(); + Attributes commitLogAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase()) + .build(); - Attributes consumeQueueAttributes = newAttributesBuilder() - .put(LABEL_TOPIC, mq.getTopic()) - .put(LABEL_QUEUE_ID, mq.getQueueId()) - .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) - .build(); - long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset(); - long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset); - if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) { - measurement.record(0, consumeQueueAttributes); - } else { - measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes); + Attributes consumeQueueAttributes = newAttributesBuilder() + .put(LABEL_TOPIC, mq.getTopic()) + .put(LABEL_QUEUE_ID, mq.getQueueId()) + .put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase()) + .build(); + long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset(); + long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset); + if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) { + measurement.record(0, consumeQueueAttributes); + } else { + measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes); + } + } catch (ConsumeQueueException e) { + // TODO: handle exception } } }); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 05d88f7b00..db19d34405 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -40,6 +40,7 @@ java_library( "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", "@maven//:io_github_aliyunmq_rocketmq_logback_classic", "@maven//:org_apache_rocketmq_rocketmq_rocksdb", + "@maven//:com_alibaba_fastjson2_fastjson2", ], ) @@ -56,7 +57,8 @@ java_library( "//:test_deps", "@maven//:org_apache_commons_commons_lang3", "@maven//:io_netty_netty_all", - "@maven//:commons_cli_commons_cli", + "@maven//:commons_cli_commons_cli", + "@maven//:org_junit_jupiter_junit_jupiter_api", ], resources = glob(["src/test/resources/*.xml"]), )