opensource/rocketmq
3
0
Fork 0
mirror of https://github.com/apache/rocketmq.git synced 2026-08-28 11:22:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support LMQ dispatch in case if Consume Queue Store is RocksDB-based (#8842)

Browse Source 
* feat: support LMQ dispatch

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: introduce group-commit for batch insertion of RocksDB KV pairs

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: propagate store error to broker module

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* chore: fix all Bazel warning and errors

Signed-off-by: Zhanhui Li <lizhanhui@gmail.com>

* fix: remove unnecessary batch-ops when writing RocksDB using atomic flush

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: find a writable directory for RocksDB logs

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* chore: clean up ConfigHelperTest

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: truncate consume queues in case commit log records are truncated

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: truncate LMQ max offsets

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: correct truncate boundary of consume queues

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* fix: correct MessageExt encoding

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

* chore: remove unused import

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>

---------

Signed-off-by: Li Zhanhui <lizhanhui@gmail.com>
Signed-off-by: Zhanhui Li <lizhanhui@gmail.com>
This commit is contained in:
Zhanhui Li
2024-10-23 09:56:37 +08:00
committed by GitHub
parent d2fd068be7
commit b86059c7c1
68 changed files with 1450 additions and 824 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2 -1
View File
@@ -17,4 +17,5 @@ bazel-out
bazel-bin
bazel-rocketmq
bazel-testlogs
.vscode
.vscode
MODULE.bazel.lock
MODULE.bazel
+22
View File
@@ -0,0 +1,22 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
# Bazel now uses Bzlmod by default to manage external dependencies.
# Please consider migrating your external dependencies from WORKSPACE to MODULE.bazel.
#
# For more details, please check https://github.com/bazelbuild/bazel/issues/18958
###############################################################################
WORKSPACE
+2
View File
@@ -112,6 +112,8 @@ maven_install(
"com.alipay.sofa:hessian:3.3.6",
"io.netty:netty-tcnative-boringssl-static:2.0.48.Final",
"org.mockito:mockito-junit-jupiter:4.11.0",
"com.alibaba.fastjson2:fastjson2:2.0.43",
"org.junit.jupiter:junit-jupiter-api:5.9.1",
],
fetch_sources = True,
repositories = [
broker/BUILD.bazel
+3
View File
@@ -91,6 +91,9 @@ java_library(
"@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
"@maven//:org_powermock_powermock_core",
"@maven//:io_opentelemetry_opentelemetry_api",
"@maven//:com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru",
"@maven//:org_apache_rocketmq_rocketmq_rocksdb",
"@maven//:commons_collections_commons_collections",
],
)
broker/src/main/java/org/apache/rocketmq/broker/client/net/Broker2Client.java
+9 -5
View File
@@ -37,6 +37,7 @@ import org.apache.rocketmq.common.message.MessageQueueForC;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
@@ -49,6 +50,7 @@ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequest
import org.apache.rocketmq.remoting.protocol.header.GetConsumerStatusRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.NotifyConsumerIdsChangedRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ResetOffsetRequestHeader;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class Broker2Client {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -100,13 +102,12 @@ public class Broker2Client {
}
}
public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) {
public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) throws RemotingCommandException {
return resetOffset(topic, group, timeStamp, isForce, false);
}
public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce,
boolean isC) {
boolean isC) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -135,8 +136,11 @@ public class Broker2Client {
long timeStampOffset;
if (timeStamp == -1) {
timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
try {
timeStampOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
} else {
timeStampOffset = this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp);
}
broker/src/main/java/org/apache/rocketmq/broker/longpolling/LmqPullRequestHoldService.java
+1 -2
View File
@@ -22,7 +22,6 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
public class LmqPullRequestHoldService extends PullRequestHoldService {
private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -48,8 +47,8 @@ public class LmqPullRequestHoldService extends PullRequestHoldService {
}
String topic = key.substring(0, idx);
int queueId = Integer.parseInt(key.substring(idx + 1));
final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
final long offset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
LOGGER.error("check hold request failed. topic={}, queueId={}", topic, queueId, e);
broker/src/main/java/org/apache/rocketmq/broker/longpolling/PullRequestHoldService.java
+8 -2
View File
@@ -28,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.ConsumeQueueExt;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class PullRequestHoldService extends ServiceThread {
private static final Logger log = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME);
@@ -103,8 +104,8 @@ public class PullRequestHoldService extends ServiceThread {
if (2 == kArray.length) {
String topic = kArray[0];
int queueId = Integer.parseInt(kArray[1]);
final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
final long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
this.notifyMessageArriving(topic, queueId, offset);
} catch (Throwable e) {
log.error(
@@ -131,7 +132,12 @@ public class PullRequestHoldService extends ServiceThread {
for (PullRequest request : requestList) {
long newestOffset = maxOffset;
if (newestOffset <= request.getPullFromThisOffset()) {
newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
try {
newestOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
} catch (ConsumeQueueException e) {
log.error("Failed tp get max offset in queue", e);
continue;
}
}
if (newestOffset > request.getPullFromThisOffset()) {
broker/src/main/java/org/apache/rocketmq/broker/metrics/ConsumerLagCalculator.java
+69 -41
View File
@@ -48,6 +48,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SimpleSubscriptionData
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.store.DefaultMessageFilter;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class ConsumerLagCalculator {
private final BrokerConfig brokerConfig;
@@ -212,22 +213,30 @@ public class ConsumerLagCalculator {
CalculateLagResult result = new CalculateLagResult(info.group, info.topic, false);
Pair<Long, Long> lag = getConsumerLagStats(info.group, info.topic, info.isPop);
if (lag != null) {
result.lag = lag.getObject1();
result.earliestUnconsumedTimestamp = lag.getObject2();
}
lagRecorder.accept(result);
if (info.isPop) {
Pair<Long, Long> retryLag = getConsumerLagStats(info.group, info.retryTopic, true);
result = new CalculateLagResult(info.group, info.topic, true);
if (retryLag != null) {
result.lag = retryLag.getObject1();
result.earliestUnconsumedTimestamp = retryLag.getObject2();
try {
Pair<Long, Long> lag = getConsumerLagStats(info.group, info.topic, info.isPop);
if (lag != null) {
result.lag = lag.getObject1();
result.earliestUnconsumedTimestamp = lag.getObject2();
}
lagRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get lag stats", e);
}
if (info.isPop) {
try {
Pair<Long, Long> retryLag = getConsumerLagStats(info.group, info.retryTopic, true);
result = new CalculateLagResult(info.group, info.topic, true);
if (retryLag != null) {
result.lag = retryLag.getObject1();
result.earliestUnconsumedTimestamp = retryLag.getObject2();
}
lagRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get lag stats", e);
}
}
});
}
@@ -235,22 +244,30 @@ public class ConsumerLagCalculator {
public void calculateInflight(Consumer<CalculateInflightResult> inflightRecorder) {
processAllGroup(info -> {
CalculateInflightResult result = new CalculateInflightResult(info.group, info.topic, false);
Pair<Long, Long> inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
if (inFlight != null) {
result.inFlight = inFlight.getObject1();
result.earliestUnPulledTimestamp = inFlight.getObject2();
}
inflightRecorder.accept(result);
if (info.isPop) {
Pair<Long, Long> retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
result = new CalculateInflightResult(info.group, info.topic, true);
if (retryInFlight != null) {
result.inFlight = retryInFlight.getObject1();
result.earliestUnPulledTimestamp = retryInFlight.getObject2();
try {
Pair<Long, Long> inFlight = getInFlightMsgStats(info.group, info.topic, info.isPop);
if (inFlight != null) {
result.inFlight = inFlight.getObject1();
result.earliestUnPulledTimestamp = inFlight.getObject2();
}
inflightRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get inflight message stats", e);
}
if (info.isPop) {
try {
Pair<Long, Long> retryInFlight = getInFlightMsgStats(info.group, info.retryTopic, true);
result = new CalculateInflightResult(info.group, info.topic, true);
if (retryInFlight != null) {
result.inFlight = retryInFlight.getObject1();
result.earliestUnPulledTimestamp = retryInFlight.getObject2();
}
inflightRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get inflight message stats", e);
}
}
});
}
@@ -259,20 +276,28 @@ public class ConsumerLagCalculator {
processAllGroup(info -> {
CalculateAvailableResult result = new CalculateAvailableResult(info.group, info.topic, false);
result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
availableRecorder.accept(result);
try {
result.available = getAvailableMsgCount(info.group, info.topic, info.isPop);
availableRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get available message count", e);
}
if (info.isPop) {
long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
result = new CalculateAvailableResult(info.group, info.topic, true);
result.available = retryAvailable;
availableRecorder.accept(result);
try {
long retryAvailable = getAvailableMsgCount(info.group, info.retryTopic, true);
result = new CalculateAvailableResult(info.group, info.topic, true);
result.available = retryAvailable;
availableRecorder.accept(result);
} catch (ConsumeQueueException e) {
LOGGER.error("Failed to get available message count", e);
}
}
});
}
public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean isPop) {
public Pair<Long, Long> getConsumerLagStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
long earliestUnconsumedTimestamp = Long.MAX_VALUE;
@@ -298,7 +323,8 @@ public class ConsumerLagCalculator {
return new Pair<>(total, earliestUnconsumedTimestamp);
}
public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queueId, boolean isPop) {
public Pair<Long, Long> getConsumerLagStats(String group, String topic, int queueId, boolean isPop)
throws ConsumeQueueException {
long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
if (brokerOffset < 0) {
brokerOffset = 0;
@@ -329,7 +355,7 @@ public class ConsumerLagCalculator {
return new Pair<>(lag, consumerStoreTimeStamp);
}
public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean isPop) {
public Pair<Long, Long> getInFlightMsgStats(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
long earliestUnPulledTimestamp = Long.MAX_VALUE;
@@ -355,7 +381,8 @@ public class ConsumerLagCalculator {
return new Pair<>(total, earliestUnPulledTimestamp);
}
public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queueId, boolean isPop) {
public Pair<Long, Long> getInFlightMsgStats(String group, String topic, int queueId, boolean isPop)
throws ConsumeQueueException {
if (isPop) {
long inflight = popInflightMessageCounter.getGroupPopInFlightMessageNum(topic, group, queueId);
long pullOffset = popBufferMergeService.getLatestOffset(topic, group, queueId);
@@ -384,7 +411,7 @@ public class ConsumerLagCalculator {
return new Pair<>(inflight, pullStoreTimeStamp);
}
public long getAvailableMsgCount(String group, String topic, boolean isPop) {
public long getAvailableMsgCount(String group, String topic, boolean isPop) throws ConsumeQueueException {
long total = 0L;
if (group == null || topic == null) {
@@ -403,7 +430,8 @@ public class ConsumerLagCalculator {
return total;
}
public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop) {
public long getAvailableMsgCount(String group, String topic, int queueId, boolean isPop)
throws ConsumeQueueException {
long brokerOffset = messageStore.getMaxOffsetInQueue(topic, queueId);
if (brokerOffset < 0) {
brokerOffset = 0;
broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java
+18 -6
View File
@@ -39,8 +39,11 @@ import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.metrics.NopLongCounter;
import org.apache.rocketmq.common.metrics.NopLongHistogram;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_TOPIC;
@@ -57,6 +60,7 @@ import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_QUEUE_
import static org.apache.rocketmq.broker.metrics.PopMetricsConstant.LABEL_REVIVE_MESSAGE_TYPE;
public class PopMetricsManager {
private static final Logger log = LoggerFactory.getLogger(PopMetricsManager.class);
public static Supplier<AttributesBuilder> attributesBuilderSupplier;
private static LongHistogram popBufferScanTimeConsume = new NopLongHistogram();
@@ -138,9 +142,13 @@ public class PopMetricsManager {
ObservableLongMeasurement measurement) {
PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
for (PopReviveService popReviveService : popReviveServices) {
measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
.put(LABEL_QUEUE_ID, popReviveService.getQueueId())
.build());
try {
measurement.record(popReviveService.getReviveBehindMillis(), newAttributesBuilder()
.put(LABEL_QUEUE_ID, popReviveService.getQueueId())
.build());
} catch (ConsumeQueueException e) {
log.error("Failed to get revive behind duration", e);
}
}
}
@@ -148,9 +156,13 @@ public class PopMetricsManager {
ObservableLongMeasurement measurement) {
PopReviveService[] popReviveServices = brokerController.getAckMessageProcessor().getPopReviveServices();
for (PopReviveService popReviveService : popReviveServices) {
measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
.put(LABEL_QUEUE_ID, popReviveService.getQueueId())
.build());
try {
measurement.record(popReviveService.getReviveBehindMessages(), newAttributesBuilder()
.put(LABEL_QUEUE_ID, popReviveService.getQueueId())
.build());
} catch (ConsumeQueueException e) {
log.error("Failed to get revive behind message count", e);
}
}
}
broker/src/main/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManager.java
+17 -19
View File
@@ -25,6 +25,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
/**
* manage the offset of broadcast.
@@ -72,7 +73,7 @@ public class BroadcastOffsetManager extends ServiceThread {
* @return -1 means no init offset, use the queueOffset in pullRequestHeader
*/
public Long queryInitOffset(String topic, String groupId, int queueId, String clientId, long requestOffset,
boolean fromProxy) {
boolean fromProxy) throws ConsumeQueueException {
BroadcastOffsetData broadcastOffsetData = offsetStoreMap.get(buildKey(topic, groupId));
if (broadcastOffsetData == null) {
@@ -84,29 +85,26 @@ public class BroadcastOffsetManager extends ServiceThread {
}
final AtomicLong offset = new AtomicLong(-1L);
broadcastOffsetData.clientOffsetStore.compute(clientId, (clientIdK, offsetStore) -> {
if (offsetStore == null) {
offsetStore = new BroadcastTimedOffsetStore(fromProxy);
}
if (offsetStore.fromProxy && requestOffset < 0) {
// when from proxy and requestOffset is -1
// means proxy need a init offset to pull message
offset.set(getOffset(offsetStore, topic, groupId, queueId));
return offsetStore;
}
if (offsetStore.fromProxy == fromProxy) {
return offsetStore;
}
BroadcastTimedOffsetStore offsetStore = broadcastOffsetData.clientOffsetStore.get(clientId);
if (offsetStore == null) {
offsetStore = new BroadcastTimedOffsetStore(fromProxy);
broadcastOffsetData.clientOffsetStore.put(clientId, offsetStore);
}
if (offsetStore.fromProxy && requestOffset < 0) {
// when from proxy and requestOffset is -1
// means proxy need a init offset to pull message
offset.set(getOffset(offsetStore, topic, groupId, queueId));
return offsetStore;
});
} else {
if (offsetStore.fromProxy != fromProxy) {
offset.set(getOffset(offsetStore, topic, groupId, queueId));
}
}
return offset.get();
}
private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId) {
private long getOffset(BroadcastTimedOffsetStore offsetStore, String topic, String groupId, int queueId)
throws ConsumeQueueException {
long storeOffset = -1;
if (offsetStore != null) {
storeOffset = offsetStore.offsetStore.readOffset(queueId);
broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java
+16 -5
View File
@@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.util.BitSet;
import java.nio.charset.StandardCharsets;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.metrics.PopMetricsManager;
import org.apache.rocketmq.common.KeyBuilder;
@@ -30,7 +31,6 @@ import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -45,6 +45,7 @@ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
@@ -134,7 +135,12 @@ public class AckMessageProcessor implements NettyRequestProcessor {
}
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
long maxOffset;
try {
maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset", e);
}
if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
String errorInfo = String.format("offset is illegal, key:%s@%d, commit:%d, store:%d~%d",
requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getOffset(), minOffset, maxOffset);
@@ -166,7 +172,7 @@ public class AckMessageProcessor implements NettyRequestProcessor {
}
private void appendAck(final AckMessageRequestHeader requestHeader, final BatchAck batchAck,
final RemotingCommand response, final Channel channel, String brokerName) {
final RemotingCommand response, final Channel channel, String brokerName) throws RemotingCommandException {
String[] extraInfo;
String consumeGroup, topic;
int qId, rqId;
@@ -206,7 +212,12 @@ public class AckMessageProcessor implements NettyRequestProcessor {
invisibleTime = batchAck.getInvisibleTime();
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, qId);
long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
long maxOffset;
try {
maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, qId);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
if (minOffset == -1 || maxOffset == -1) {
POP_LOGGER.error("Illegal topic or queue found when batch ack {}", batchAck);
return;
@@ -254,7 +265,7 @@ public class AckMessageProcessor implements NettyRequestProcessor {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(reviveTopic);
msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(rqId);
if (ackMsg instanceof BatchAckMsg) {
msgInner.setTags(PopAckConstants.BATCH_ACK_TAG);
broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java
+167 -133
View File
@@ -215,6 +215,7 @@ import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.RocksDBMessageStore;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.plugin.AbstractPluginMessageStore;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
@@ -1341,8 +1342,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class);
final GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader();
final GetMaxOffsetRequestHeader requestHeader =
(GetMaxOffsetRequestHeader) request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
final GetMaxOffsetRequestHeader requestHeader = request.decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class);
TopicQueueMappingContext mappingContext = this.brokerController.getTopicQueueMappingManager().buildTopicQueueMappingContext(requestHeader);
RemotingCommand rewriteResult = rewriteRequestForStaticTopic(requestHeader, mappingContext);
@@ -1350,10 +1350,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return rewriteResult;
}
long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
responseHeader.setOffset(offset);
try {
long offset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
responseHeader.setOffset(offset);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
@@ -1484,7 +1486,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return response;
}
private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) {
private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
HashMap<String, String> runtimeInfo = this.prepareRuntimeInfo();
@@ -1686,8 +1689,8 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return response;
}
private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig) {
private void initConsumerOffset(String clientHost, String groupName, int mode, TopicConfig topicConfig)
throws ConsumeQueueException {
String topic = topicConfig.getTopicName();
for (int queueId = 0; queueId < topicConfig.getReadQueueNums(); queueId++) {
if (this.brokerController.getConsumerOffsetManager().queryOffset(groupName, topic, queueId) > -1) {
@@ -1761,8 +1764,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final GetTopicStatsInfoRequestHeader requestHeader =
(GetTopicStatsInfoRequestHeader) request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
final GetTopicStatsInfoRequestHeader requestHeader = request.decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class);
final String topic = requestHeader.getTopic();
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -1775,39 +1777,45 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
TopicStatsTable topicStatsTable = new TopicStatsTable();
int maxQueueNums = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums());
for (int i = 0; i < maxQueueNums; i++) {
MessageQueue mq = new MessageQueue();
mq.setTopic(topic);
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
try {
for (int i = 0; i < maxQueueNums; i++) {
MessageQueue mq = new MessageQueue();
mq.setTopic(topic);
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
TopicOffset topicOffset = new TopicOffset();
long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
if (min < 0) {
min = 0;
TopicOffset topicOffset = new TopicOffset();
long min = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, i);
if (min < 0) {
min = 0;
}
long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (max < 0) {
max = 0;
}
long timestamp = 0;
if (max > 0) {
timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
}
topicOffset.setMinOffset(min);
topicOffset.setMaxOffset(max);
topicOffset.setLastUpdateTimestamp(timestamp);
topicStatsTable.getOffsetTable().put(mq, topicOffset);
}
long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (max < 0) {
max = 0;
}
long timestamp = 0;
if (max > 0) {
timestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
}
topicOffset.setMinOffset(min);
topicOffset.setMaxOffset(max);
topicOffset.setLastUpdateTimestamp(timestamp);
topicStatsTable.getOffsetTable().put(mq, topicOffset);
byte[] body = topicStatsTable.encode();
response.setBody(body);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
} catch (ConsumeQueueException e) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(e.getMessage());
}
byte[] body = topicStatsTable.encode();
response.setBody(body);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
}
@@ -1907,93 +1915,96 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
private RemotingCommand getConsumeStats(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
final GetConsumeStatsRequestHeader requestHeader =
(GetConsumeStatsRequestHeader) request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
try {
final GetConsumeStatsRequestHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class);
ConsumeStats consumeStats = new ConsumeStats();
ConsumeStats consumeStats = new ConsumeStats();
Set<String> topics = new HashSet<>();
if (UtilAll.isBlank(requestHeader.getTopic())) {
topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
} else {
topics.add(requestHeader.getTopic());
}
for (String topic : topics) {
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
if (null == topicConfig) {
LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
continue;
Set<String> topics = new HashSet<>();
if (UtilAll.isBlank(requestHeader.getTopic())) {
topics = this.brokerController.getConsumerOffsetManager().whichTopicByConsumer(requestHeader.getConsumerGroup());
} else {
topics.add(requestHeader.getTopic());
}
TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
{
SubscriptionData findSubscriptionData =
this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
if (null == findSubscriptionData
&& this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
LOGGER.warn(
"AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, "
+ "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup());
for (String topic : topics) {
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
if (null == topicConfig) {
LOGGER.warn("AdminBrokerProcessor#getConsumeStats: topic config does not exist, topic={}", topic);
continue;
}
}
for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
MessageQueue mq = new MessageQueue();
mq.setTopic(topic);
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
TopicQueueMappingDetail mappingDetail = this.brokerController.getTopicQueueMappingManager().getTopicQueueMapping(topic);
OffsetWrapper offsetWrapper = new OffsetWrapper();
{
SubscriptionData findSubscriptionData =
this.brokerController.getConsumerManager().findSubscriptionData(requestHeader.getConsumerGroup(), topic);
long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (brokerOffset < 0) {
brokerOffset = 0;
}
long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
requestHeader.getConsumerGroup(), topic, i);
// the consumerOffset cannot be zero for static topic because of the "double read check" strategy
// just remain the logic for dynamic topic
// maybe we should remove it in the future
if (mappingDetail == null) {
if (consumerOffset < 0) {
consumerOffset = 0;
if (null == findSubscriptionData
&& this.brokerController.getConsumerManager().findSubscriptionDataCount(requestHeader.getConsumerGroup()) > 0) {
LOGGER.warn(
"AdminBrokerProcessor#getConsumeStats: topic does not exist in consumer group's subscription, "
+ "topic={}, consumer group={}", topic, requestHeader.getConsumerGroup());
continue;
}
}
long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
requestHeader.getConsumerGroup(), topic, i);
for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
MessageQueue mq = new MessageQueue();
mq.setTopic(topic);
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
offsetWrapper.setBrokerOffset(brokerOffset);
offsetWrapper.setConsumerOffset(consumerOffset);
offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
OffsetWrapper offsetWrapper = new OffsetWrapper();
long timeOffset = consumerOffset - 1;
if (timeOffset >= 0) {
long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
if (lastTimestamp > 0) {
offsetWrapper.setLastTimestamp(lastTimestamp);
long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
if (brokerOffset < 0) {
brokerOffset = 0;
}
long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(
requestHeader.getConsumerGroup(), topic, i);
// the consumerOffset cannot be zero for static topic because of the "double read check" strategy
// just remain the logic for dynamic topic
// maybe we should remove it in the future
if (mappingDetail == null) {
if (consumerOffset < 0) {
consumerOffset = 0;
}
}
long pullOffset = this.brokerController.getConsumerOffsetManager().queryPullOffset(
requestHeader.getConsumerGroup(), topic, i);
offsetWrapper.setBrokerOffset(brokerOffset);
offsetWrapper.setConsumerOffset(consumerOffset);
offsetWrapper.setPullOffset(Math.max(consumerOffset, pullOffset));
long timeOffset = consumerOffset - 1;
if (timeOffset >= 0) {
long lastTimestamp = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, timeOffset);
if (lastTimestamp > 0) {
offsetWrapper.setLastTimestamp(lastTimestamp);
}
}
consumeStats.getOffsetTable().put(mq, offsetWrapper);
}
consumeStats.getOffsetTable().put(mq, offsetWrapper);
double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
consumeTps += consumeStats.getConsumeTps();
consumeStats.setConsumeTps(consumeTps);
}
double consumeTps = this.brokerController.getBrokerStatsManager().tpsGroupGetNums(requestHeader.getConsumerGroup(), topic);
consumeTps += consumeStats.getConsumeTps();
consumeStats.setConsumeTps(consumeTps);
byte[] body = consumeStats.encode();
response.setBody(body);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
} catch (ConsumeQueueException e) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(e.getMessage());
}
byte[] body = consumeStats.encode();
response.setBody(body);
response.setCode(ResponseCode.SUCCESS);
response.setRemark(null);
return response;
}
@@ -2108,7 +2119,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
requestHeader.getTimestamp(), requestHeader.isForce(), isC);
}
private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) {
private Long searchOffsetByTimestamp(String topic, int queueId, long timestamp) throws ConsumeQueueException {
if (timestamp < 0) {
return brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
} else {
@@ -2155,25 +2166,31 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return response;
}
if (queueId >= 0) {
if (null != offset && -1 != offset) {
long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
if (min >= 0 && offset < min || offset > max + 1) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(
String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
return response;
try {
if (queueId >= 0) {
if (null != offset && -1 != offset) {
long min = brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
long max = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
if (min >= 0 && offset < min || offset > max + 1) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(
String.format("Target offset %d not in consume queue range [%d-%d]", offset, min, max));
return response;
}
} else {
offset = searchOffsetByTimestamp(topic, queueId, timestamp);
}
queueOffsetMap.put(queueId, offset);
} else {
offset = searchOffsetByTimestamp(topic, queueId, timestamp);
}
queueOffsetMap.put(queueId, offset);
} else {
for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
offset = searchOffsetByTimestamp(topic, index, timestamp);
queueOffsetMap.put(index, offset);
for (int index = 0; index < topicConfig.getReadQueueNums(); index++) {
offset = searchOffsetByTimestamp(topic, index, timestamp);
queueOffsetMap.put(index, offset);
}
}
} catch (ConsumeQueueException e) {
response.setCode(ResponseCode.SYSTEM_ERROR);
response.setRemark(e.getMessage());
return response;
}
if (queueOffsetMap.isEmpty()) {
@@ -2280,8 +2297,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx,
RemotingCommand request) throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
QueryConsumeTimeSpanRequestHeader requestHeader =
(QueryConsumeTimeSpanRequestHeader) request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
QueryConsumeTimeSpanRequestHeader requestHeader = request.decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class);
final String topic = requestHeader.getTopic();
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic);
@@ -2303,7 +2319,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i);
timeSpan.setMinTimeStamp(minTime);
long max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
long max;
try {
max = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
long maxTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, max - 1);
timeSpan.setMaxTimeStamp(maxTime);
@@ -2317,7 +2338,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
}
timeSpan.setConsumeTimeStamp(consumeTime);
long maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
long maxBrokerOffset;
try {
maxBrokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), i);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
if (consumerOffset < maxBrokerOffset) {
long nextTime = this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, consumerOffset);
timeSpan.setDelayTime(System.currentTimeMillis() - nextTime);
@@ -2552,8 +2578,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
private RemotingCommand fetchAllConsumeStatsInBroker(ChannelHandlerContext ctx, RemotingCommand request)
throws RemotingCommandException {
final RemotingCommand response = RemotingCommand.createResponseCommand(null);
GetConsumeStatsInBrokerHeader requestHeader =
(GetConsumeStatsInBrokerHeader) request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
GetConsumeStatsInBrokerHeader requestHeader = request.decodeCommandCustomHeader(GetConsumeStatsInBrokerHeader.class);
boolean isOrder = requestHeader.isOrder();
ConcurrentMap<String, SubscriptionGroupConfig> subscriptionGroups =
brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable();
@@ -2599,7 +2624,12 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName());
mq.setQueueId(i);
OffsetWrapper offsetWrapper = new OffsetWrapper();
long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
long brokerOffset;
try {
brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, i);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset", e);
}
if (brokerOffset < 0) {
brokerOffset = 0;
}
@@ -2643,7 +2673,7 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
return response;
}
private HashMap<String, String> prepareRuntimeInfo() {
private HashMap<String, String> prepareRuntimeInfo() throws RemotingCommandException {
HashMap<String, String> runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo();
for (BrokerAttachedPlugin brokerAttachedPlugin : brokerController.getBrokerAttachedPlugins()) {
@@ -2652,7 +2682,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor {
}
}
this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
try {
this.brokerController.getScheduleMessageService().buildRunningStats(runtimeInfo);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
runtimeInfo.put("brokerActive", String.valueOf(this.brokerController.isSpecialServiceRunning()));
runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CURRENT_VERSION));
runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CURRENT_VERSION));
broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java
+10 -4
View File
@@ -21,6 +21,7 @@ import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.nio.charset.StandardCharsets;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.metrics.PopMetricsManager;
import org.apache.rocketmq.common.PopAckConstants;
@@ -30,7 +31,6 @@ import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -43,6 +43,7 @@ import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeRequestHe
import org.apache.rocketmq.remoting.protocol.header.ChangeInvisibleTimeResponseHeader;
import org.apache.rocketmq.remoting.protocol.header.ExtraInfoUtil;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -120,7 +121,12 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
return CompletableFuture.completedFuture(response);
}
long minOffset = this.brokerController.getMessageStore().getMinOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
long maxOffset;
try {
maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(requestHeader.getTopic(), requestHeader.getQueueId());
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to get max consume offset", e);
}
if (requestHeader.getOffset() < minOffset || requestHeader.getOffset() > maxOffset) {
response.setCode(ResponseCode.NO_MESSAGE);
return CompletableFuture.completedFuture(response);
@@ -201,7 +207,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
}
msgInner.setTopic(reviveTopic);
msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8));
msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(rqId);
msgInner.setTags(PopAckConstants.ACK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
@@ -244,7 +250,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
ck.addDiff(0);
ck.setBrokerName(ExtraInfoUtil.getBrokerName(extraInfo));
msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(reviveQid);
msgInner.setTags(PopAckConstants.CK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
broker/src/main/java/org/apache/rocketmq/broker/processor/NotificationProcessor.java
+12 -5
View File
@@ -41,6 +41,7 @@ import org.apache.rocketmq.remoting.protocol.ResponseCode;
import org.apache.rocketmq.remoting.protocol.header.NotificationRequestHeader;
import org.apache.rocketmq.remoting.protocol.header.NotificationResponseHeader;
import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class NotificationProcessor implements NettyRequestProcessor {
private static final Logger POP_LOGGER = LoggerFactory.getLogger(LoggerName.ROCKETMQ_POP_LOGGER_NAME);
@@ -169,13 +170,15 @@ public class NotificationProcessor implements NettyRequestProcessor {
return response;
}
private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader) {
private boolean hasMsgFromTopic(String topicName, int randomQ, NotificationRequestHeader requestHeader)
throws RemotingCommandException {
boolean hasMsg;
TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topicName);
return hasMsgFromTopic(topicConfig, randomQ, requestHeader);
}
private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader) {
private boolean hasMsgFromTopic(TopicConfig topicConfig, int randomQ, NotificationRequestHeader requestHeader)
throws RemotingCommandException {
boolean hasMsg;
if (topicConfig != null) {
for (int i = 0; i < topicConfig.getReadQueueNums(); i++) {
@@ -189,15 +192,19 @@ public class NotificationProcessor implements NettyRequestProcessor {
return false;
}
private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) {
private boolean hasMsgFromQueue(String targetTopic, NotificationRequestHeader requestHeader, int queueId) throws RemotingCommandException {
if (Boolean.TRUE.equals(requestHeader.getOrder())) {
if (this.brokerController.getConsumerOrderInfoManager().checkBlock(requestHeader.getAttemptId(), requestHeader.getTopic(), requestHeader.getConsumerGroup(), queueId, 0)) {
return false;
}
}
long offset = getPopOffset(targetTopic, requestHeader.getConsumerGroup(), queueId);
long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
return restNum > 0;
try {
long restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(targetTopic, queueId) - offset;
return restNum > 0;
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed tp get max offset in queue", e);
}
}
private long getPopOffset(String topic, String cid, int queueId) {
broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java
+8 -2
View File
@@ -51,6 +51,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_CONSUMER_GROUP;
import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.LABEL_IS_SYSTEM;
@@ -229,13 +230,18 @@ public class PeekMessageProcessor implements NettyRequestProcessor {
private long peekMsgFromQueue(boolean isRetry, GetMessageResult getMessageResult,
PeekMessageRequestHeader requestHeader, int queueId, long restNum, int reviveQid, Channel channel,
long popTime) {
long popTime) throws RemotingCommandException {
String topic = isRetry ?
KeyBuilder.buildPopRetryTopic(requestHeader.getTopic(), requestHeader.getConsumerGroup(), brokerController.getBrokerConfig().isEnableRetryTopicV2())
: requestHeader.getTopic();
GetMessageResult getMessageTmpResult;
long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId);
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
try {
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
} catch (ConsumeQueueException e) {
LOG.error("Failed to get max offset in queue. topic={}, queue-id={}", topic, queueId, e);
throw new RemotingCommandException("Failed to get max offset in queue", e);
}
if (getMessageResult.getMessageMapedList().size() >= requestHeader.getMaxMsgNums()) {
return restNum;
}
broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java
+39 -16
View File
@@ -24,6 +24,7 @@ import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.FileRegion;
import io.opentelemetry.api.common.Attributes;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -62,7 +63,6 @@ import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.remoting.common.RemotingHelper;
@@ -83,6 +83,7 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -167,13 +168,14 @@ public class PopMessageProcessor implements NettyRequestProcessor {
return popLongPollingService.getPollingMap();
}
public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) {
public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId) throws ConsumeQueueException {
this.notifyLongPollingRequestIfNeed(
topic, group, queueId, null, 0L, null, null);
}
public void notifyLongPollingRequestIfNeed(String topic, String group, int queueId,
Long tagsCode, long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
Long tagsCode, long msgStoreTime, byte[] filterBitMap,
Map<String, String> properties) throws ConsumeQueueException {
long popBufferOffset = this.brokerController.getPopMessageProcessor().getPopBufferMergeService().getLatestOffset(topic, group, queueId);
long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
long maxOffset = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
@@ -217,8 +219,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class);
final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader();
final PopMessageRequestHeader requestHeader =
(PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
final PopMessageRequestHeader requestHeader = request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true);
StringBuilder startOffsetInfo = new StringBuilder(64);
StringBuilder msgOffsetInfo = new StringBuilder(64);
StringBuilder orderCountInfo = null;
@@ -531,20 +532,37 @@ public class PopMessageProcessor implements NettyRequestProcessor {
String lockKey =
topic + PopAckConstants.SPLIT + requestHeader.getConsumerGroup() + PopAckConstants.SPLIT + queueId;
boolean isOrder = requestHeader.isOrder();
long offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
false, lockKey, false);
long offset;
try {
offset = getPopOffset(topic, requestHeader.getConsumerGroup(), queueId, requestHeader.getInitMode(),
false, lockKey, false);
} catch (ConsumeQueueException e) {
CompletableFuture<Long> failure = new CompletableFuture<>();
failure.completeExceptionally(e);
return failure;
}
CompletableFuture<Long> future = new CompletableFuture<>();
if (!queueLockManager.tryLock(lockKey)) {
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
future.complete(restNum);
try {
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
future.complete(restNum);
} catch (ConsumeQueueException e) {
future.completeExceptionally(e);
}
return future;
}
future.whenComplete((result, throwable) -> queueLockManager.unLock(lockKey));
if (isPopShouldStop(topic, requestHeader.getConsumerGroup(), queueId)) {
POP_LOGGER.warn("Too much msgs unacked, then stop poping. topic={}, group={}, queueId={}", topic, requestHeader.getConsumerGroup(), queueId);
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
future.complete(restNum);
POP_LOGGER.warn("Too much msgs unacked, then stop popping. topic={}, group={}, queueId={}",
topic, requestHeader.getConsumerGroup(), queueId);
try {
restNum = this.brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - offset + restNum;
future.complete(restNum);
} catch (ConsumeQueueException e) {
future.completeExceptionally(e);
}
return future;
}
@@ -610,7 +628,11 @@ public class PopMessageProcessor implements NettyRequestProcessor {
return CompletableFuture.completedFuture(result);
}).thenApply(result -> {
if (result == null) {
atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
try {
atomicRestNum.set(brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId) - atomicOffset.get() + atomicRestNum.get());
} catch (ConsumeQueueException e) {
POP_LOGGER.error("Failed to get max offset in queue", e);
}
return atomicRestNum.get();
}
if (!result.getMessageMapedList().isEmpty()) {
@@ -710,7 +732,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
}
private long getPopOffset(String topic, String group, int queueId, int initMode, boolean init, String lockKey,
boolean checkResetOffset) {
boolean checkResetOffset) throws ConsumeQueueException {
long offset = this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, queueId);
if (offset < 0) {
@@ -732,7 +754,8 @@ public class PopMessageProcessor implements NettyRequestProcessor {
}
}
private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init) {
private long getInitOffset(String topic, String group, int queueId, int initMode, boolean init)
throws ConsumeQueueException {
long offset;
if (ConsumeInitMode.MIN == initMode || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
offset = this.brokerController.getMessageStore().getMinOffsetInQueue(topic, queueId);
@@ -761,7 +784,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
MessageExtBrokerInner msgInner = new MessageExtBrokerInner();
msgInner.setTopic(reviveTopic);
msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8));
msgInner.setBody(JSON.toJSONString(ck).getBytes(StandardCharsets.UTF_8));
msgInner.setQueueId(reviveQid);
msgInner.setTags(PopAckConstants.CK_TAG);
msgInner.setBornTimestamp(System.currentTimeMillis());
broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java
+14 -8
View File
@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor;
import com.alibaba.fastjson.JSON;
import io.opentelemetry.api.common.Attributes;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -54,6 +55,7 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.AppendMessageStatus;
import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.pop.AckMsg;
import org.apache.rocketmq.store.pop.BatchAckMsg;
import org.apache.rocketmq.store.pop.PopCheckPoint;
@@ -260,10 +262,14 @@ public class PopReviveService extends ServiceThread {
getMessageResult.getMaxOffset(), foundList);
} else {
long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
if (maxQueueOffset > offset) {
POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
topic, group, offset, maxQueueOffset);
try {
long maxQueueOffset = brokerController.getMessageStore().getMaxOffsetInQueue(topic, queueId);
if (maxQueueOffset > offset) {
POP_LOGGER.error("get message from store return null. topic={}, groupId={}, requestOffset={}, maxQueueOffset={}",
topic, group, offset, maxQueueOffset);
}
} catch (ConsumeQueueException e) {
POP_LOGGER.error("Failed to get max offset in queue", e);
}
return null;
}
@@ -364,7 +370,7 @@ public class PopReviveService extends ServiceThread {
firstRt = point.getReviveTime();
}
} else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) {
String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
}
@@ -388,7 +394,7 @@ public class PopReviveService extends ServiceThread {
}
}
} else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) {
String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8);
String raw = new String(messageExt.getBody(), StandardCharsets.UTF_8);
if (brokerController.getBrokerConfig().isEnablePopLog()) {
POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw);
}
@@ -594,7 +600,7 @@ public class PopReviveService extends ServiceThread {
brokerController.getMessageStore().putMessage(ckMsg);
}
public long getReviveBehindMillis() {
public long getReviveBehindMillis() throws ConsumeQueueException {
if (currentReviveMessageTimestamp <= 0) {
return 0;
}
@@ -605,7 +611,7 @@ public class PopReviveService extends ServiceThread {
return 0;
}
public long getReviveBehindMessages() {
public long getReviveBehindMessages() throws ConsumeQueueException {
if (currentReviveMessageTimestamp <= 0) {
return 0;
}
broker/src/main/java/org/apache/rocketmq/broker/processor/PullMessageProcessor.java
+22 -11
View File
@@ -73,6 +73,7 @@ import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.MessageFilter;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
import static org.apache.rocketmq.remoting.protocol.RemotingCommand.buildErrorResponse;
@@ -298,7 +299,8 @@ public class PullMessageProcessor implements NettyRequestProcessor {
return false;
}
private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend, boolean brokerAllowFlowCtrSuspend)
private RemotingCommand processRequest(final Channel channel, RemotingCommand request, boolean brokerAllowSuspend,
boolean brokerAllowFlowCtrSuspend)
throws RemotingCommandException {
final long beginTimeMills = this.brokerController.getMessageStore().now();
RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class);
@@ -489,7 +491,7 @@ public class PullMessageProcessor implements NettyRequestProcessor {
final MessageStore messageStore = brokerController.getMessageStore();
if (this.brokerController.getMessageStore() instanceof DefaultMessageStore) {
DefaultMessageStore defaultMessageStore = (DefaultMessageStore)this.brokerController.getMessageStore();
DefaultMessageStore defaultMessageStore = (DefaultMessageStore) this.brokerController.getMessageStore();
boolean cgNeedColdDataFlowCtr = brokerController.getColdDataCgCtrService().isCgNeedColdDataFlowCtr(requestHeader.getConsumerGroup());
if (cgNeedColdDataFlowCtr) {
boolean isMsgLogicCold = defaultMessageStore.getCommitLog()
@@ -526,7 +528,11 @@ public class PullMessageProcessor implements NettyRequestProcessor {
getMessageResult.setStatus(GetMessageStatus.OFFSET_RESET);
getMessageResult.setNextBeginOffset(resetOffset);
getMessageResult.setMinOffset(messageStore.getMinOffsetInQueue(topic, queueId));
getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
try {
getMessageResult.setMaxOffset(messageStore.getMaxOffsetInQueue(topic, queueId));
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed tp get max offset in queue", e);
}
getMessageResult.setSuggestPullingFromSlave(false);
} else {
long broadcastInitOffset = queryBroadcastPullInitOffset(topic, group, queueId, requestHeader, channel);
@@ -589,12 +595,13 @@ public class PullMessageProcessor implements NettyRequestProcessor {
/**
* Composes the header of the response message to be sent back to the client
* @param requestHeader - the header of the request message
* @param getMessageResult - the result of the GetMessage request
* @param topicSysFlag - the system flag of the topic
*
* @param requestHeader - the header of the request message
* @param getMessageResult - the result of the GetMessage request
* @param topicSysFlag - the system flag of the topic
* @param subscriptionGroupConfig - configuration of the subscription group
* @param response - the response message to be sent back to the client
* @param clientAddress - the address of the client
* @param response - the response message to be sent back to the client
* @param clientAddress - the address of the client
*/
protected void composeResponseHeader(PullMessageRequestHeader requestHeader, GetMessageResult getMessageResult,
int topicSysFlag, SubscriptionGroupConfig subscriptionGroupConfig, RemotingCommand response,
@@ -855,7 +862,7 @@ public class PullMessageProcessor implements NettyRequestProcessor {
* When pull request is not broadcast or not return -1
*/
protected long queryBroadcastPullInitOffset(String topic, String group, int queueId,
PullMessageRequestHeader requestHeader, Channel channel) {
PullMessageRequestHeader requestHeader, Channel channel) throws RemotingCommandException {
if (!this.brokerController.getBrokerConfig().isEnableBroadcastOffsetStore()) {
return -1L;
@@ -877,8 +884,12 @@ public class PullMessageProcessor implements NettyRequestProcessor {
clientId = clientChannelInfo.getClientId();
}
return this.brokerController.getBroadcastOffsetManager()
.queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
try {
return this.brokerController.getBroadcastOffsetManager()
.queryInitOffset(topic, group, queueId, clientId, requestHeader.getQueueOffset(), proxyPullBroadcast);
} catch (ConsumeQueueException e) {
throw new RemotingCommandException("Failed to query initial offset", e);
}
}
return -1L;
}
broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java
+2 -1
View File
@@ -53,6 +53,7 @@ import org.apache.rocketmq.remoting.protocol.DataVersion;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.queue.ReferredIterator;
@@ -103,7 +104,7 @@ public class ScheduleMessageService extends ConfigManager {
return delayLevel - 1;
}
public void buildRunningStats(HashMap<String, String> stats) {
public void buildRunningStats(HashMap<String, String> stats) throws ConsumeQueueException {
for (Map.Entry<Integer, Long> next : this.offsetTable.entrySet()) {
int queueId = delayLevel2QueueId(next.getKey());
long delayOffset = next.getValue();
broker/src/test/java/org/apache/rocketmq/broker/client/net/Broker2ClientTest.java
+4 -3
View File
@@ -29,6 +29,7 @@ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.remoting.RemotingServer;
import org.apache.rocketmq.remoting.exception.RemotingCommandException;
import org.apache.rocketmq.remoting.protocol.RemotingCommand;
import org.apache.rocketmq.remoting.protocol.RemotingSerializable;
import org.apache.rocketmq.remoting.protocol.ResponseCode;
@@ -125,14 +126,14 @@ public class Broker2ClientTest {
}
@Test
public void testResetOffsetNoTopicConfig() {
public void testResetOffsetNoTopicConfig() throws RemotingCommandException {
when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(null);
RemotingCommand response = broker2Client.resetOffset(defaultTopic, defaultGroup, timestamp, isForce);
assertEquals(ResponseCode.SYSTEM_ERROR, response.getCode());
}
@Test
public void testResetOffsetNoConsumerGroupInfo() {
public void testResetOffsetNoConsumerGroupInfo() throws RemotingCommandException {
TopicConfig topicConfig = mock(TopicConfig.class);
when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
when(topicConfig.getWriteQueueNums()).thenReturn(1);
@@ -142,7 +143,7 @@ public class Broker2ClientTest {
}
@Test
public void testResetOffset() {
public void testResetOffset() throws RemotingCommandException {
TopicConfig topicConfig = mock(TopicConfig.class);
when(topicConfigManager.selectTopicConfig(defaultTopic)).thenReturn(topicConfig);
when(topicConfig.getWriteQueueNums()).thenReturn(1);
broker/src/test/java/org/apache/rocketmq/broker/offset/BroadcastOffsetManagerTest.java
+4 -3
View File
@@ -25,6 +25,7 @@ import org.apache.rocketmq.broker.client.ClientChannelInfo;
import org.apache.rocketmq.broker.client.ConsumerManager;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +52,7 @@ public class BroadcastOffsetManagerTest {
private BroadcastOffsetManager broadcastOffsetManager;
@Before
public void before() {
public void before() throws ConsumeQueueException {
brokerConfig.setEnableBroadcastOffsetStore(true);
brokerConfig.setBroadcastOffsetExpireSecond(1);
brokerConfig.setBroadcastOffsetExpireMaxSecond(5);
@@ -84,7 +85,7 @@ public class BroadcastOffsetManagerTest {
}
@Test
public void testBroadcastOffsetSwitch() {
public void testBroadcastOffsetSwitch() throws ConsumeQueueException {
// client1 connect to broker
onlineClientIdSet.add("client1");
long offset = broadcastOffsetManager.queryInitOffset("group", "topic", 0, "client1", 0, false);
@@ -160,4 +161,4 @@ public class BroadcastOffsetManagerTest {
return broadcastOffsetManager.offsetStoreMap.isEmpty();
});
}
}
}
broker/src/test/java/org/apache/rocketmq/broker/offset/ConsumerOrderInfoManagerLockFreeNotifyTest.java
+3 -2
View File
@@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.rocketmq.broker.BrokerController;
import org.apache.rocketmq.broker.processor.PopMessageProcessor;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.assertj.core.util.Lists;
import org.junit.Before;
import org.junit.Test;
@@ -50,7 +51,7 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest {
private final BrokerController brokerController = mock(BrokerController.class);
@Before
public void before() {
public void before() throws ConsumeQueueException {
notified = new AtomicBoolean(false);
brokerConfig.setEnableNotifyAfterPopOrderLockRelease(true);
when(brokerController.getBrokerConfig()).thenReturn(brokerConfig);
@@ -175,4 +176,4 @@ public class ConsumerOrderInfoManagerLockFreeNotifyTest {
await().atLeast(Duration.ofSeconds(2)).atMost(Duration.ofSeconds(4)).until(notified::get);
assertTrue(consumerOrderInfoManager.getConsumerOrderInfoLockManager().getTimeoutMap().isEmpty());
}
}
}
broker/src/test/java/org/apache/rocketmq/broker/processor/AckMessageProcessorTest.java
+2 -1
View File
@@ -47,6 +47,7 @@ import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.PutMessageResult;
import org.apache.rocketmq.store.PutMessageStatus;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -93,7 +94,7 @@ public class AckMessageProcessorTest {
private static final long MAX_OFFSET_IN_QUEUE = 999;
@Before
public void init() throws IllegalAccessException, NoSuchFieldException {
public void init() throws IllegalAccessException, NoSuchFieldException, ConsumeQueueException {
clientInfo = new ClientChannelInfo(channel, "127.0.0.1", LanguageCode.JAVA, 0);
brokerController.setMessageStore(messageStore);
Field field = BrokerController.class.getDeclaredField("broker2Client");
broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java
+2 -1
View File
@@ -40,6 +40,7 @@ import org.apache.rocketmq.store.GetMessageResult;
import org.apache.rocketmq.store.GetMessageStatus;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.logfile.DefaultMappedFile;
import org.junit.Assert;
import org.junit.Before;
@@ -182,7 +183,7 @@ public class PopMessageProcessorTest {
}
@Test
public void testGetInitOffset_normalTopic() throws RemotingCommandException {
public void testGetInitOffset_normalTopic() throws RemotingCommandException, ConsumeQueueException {
long maxOffset = 999L;
when(messageStore.getMessageStoreConfig()).thenReturn(new MessageStoreConfig());
when(messageStore.getMaxOffsetInQueue(topic, 0)).thenReturn(maxOffset);
client/BUILD.bazel
+1
View File
@@ -33,6 +33,7 @@ java_library(
"@maven//:commons_collections_commons_collections",
"@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
"@maven//:io_github_aliyunmq_rocketmq_logback_classic",
"@maven//:com_google_guava_guava",
],
)
client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java
+4 -4
View File
@@ -1199,9 +1199,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdo
messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {
// process LMQ
String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)
.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
.split(MixAll.LMQ_DISPATCH_SEPARATOR);
String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)
.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
.split(MixAll.LMQ_DISPATCH_SEPARATOR);
long offset = Long.parseLong(queueOffsets[ArrayUtils.indexOf(queues, topic)]);
// LMQ topic has only 1 queue, which queue id is 0
queueIdKey = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);
@@ -1264,9 +1264,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback, StartAndShutdo
&& StringUtils.isNotEmpty(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))) {
// process LMQ
String[] queues = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)
.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
.split(MixAll.LMQ_DISPATCH_SEPARATOR);
String[] queueOffsets = messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET)
.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
.split(MixAll.LMQ_DISPATCH_SEPARATOR);
// LMQ topic has only 1 queue, which queue id is 0
key = ExtraInfoUtil.getStartOffsetInfoMapKey(topic, MixAll.LMQ_QUEUE_ID);
sortMap.putIfAbsent(key, new ArrayList<>(4));
client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java
+2 -2
View File
@@ -636,8 +636,8 @@ public class MQClientAPIImplTest {
final int invisibleTime = 10 * 1000;
final String lmqTopic = MixAll.LMQ_PREFIX + "lmq1";
final String lmqTopic2 = MixAll.LMQ_PREFIX + "lmq2";
final String multiDispatch = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, lmqTopic, lmqTopic2);
final String multiOffset = String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, "0", "0");
final String multiDispatch = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, lmqTopic, lmqTopic2);
final String multiOffset = String.join(MixAll.LMQ_DISPATCH_SEPARATOR, "0", "0");
doAnswer((Answer<Void>) mock -> {
InvokeCallback callback = mock.getArgument(3);
RemotingCommand request = mock.getArgument(1);
common/src/main/java/org/apache/rocketmq/common/MixAll.java
+9 -2
View File
@@ -48,6 +48,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.annotation.ImportantField;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.help.FAQUrl;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.common.utils.IOTinyUtils;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
@@ -99,8 +100,8 @@ public class MixAll {
public static final String ACL_CONF_TOOLS_FILE = "/conf/tools.yml";
public static final String REPLY_MESSAGE_FLAG = "reply";
public static final String LMQ_PREFIX = "%LMQ%";
public static final long LMQ_QUEUE_ID = 0;
public static final String MULTI_DISPATCH_QUEUE_SPLITTER = ",";
public static final int LMQ_QUEUE_ID = 0;
public static final String LMQ_DISPATCH_SEPARATOR = ",";
public static final String REQ_T = "ReqT";
public static final String ROCKETMQ_ZONE_ENV = "ROCKETMQ_ZONE";
public static final String ROCKETMQ_ZONE_PROPERTY = "rocketmq.zone";
@@ -524,4 +525,10 @@ public class MixAll {
}
return false;
}
public static boolean topicAllowsLMQ(String topic) {
return !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)
&& !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX)
&& !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC);
}
}
common/src/main/java/org/apache/rocketmq/common/ServiceThread.java
+1 -1
View File
@@ -24,7 +24,7 @@ import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
public abstract class ServiceThread implements Runnable {
private static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
protected static final Logger log = LoggerFactory.getLogger(LoggerName.COMMON_LOGGER_NAME);
private static final long JOIN_TIME = 90 * 1000;
common/src/main/java/org/apache/rocketmq/common/config/ConfigHelper.java
+21 -7
View File
@@ -16,8 +16,8 @@
*/
package org.apache.rocketmq.common.config;
import com.google.common.base.Strings;
import java.io.File;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.UtilAll;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.BloomFilter;
@@ -110,12 +110,26 @@ public class ConfigHelper {
}
public static String getDBLogDir() {
String rootPath = System.getProperty("user.home");
if (StringUtils.isEmpty(rootPath)) {
return "";
String[] rootPaths = new String[] {
System.getProperty("user.home"),
System.getProperty("java.io.tmpdir"),
File.separator + "data"
};
for (String rootPath : rootPaths) {
// Refer bazel test encyclopedia: https://bazel.build/reference/test-encyclopedia
// Not all directories is available
if (Strings.isNullOrEmpty(rootPath)) {
continue;
}
File rootPathFile = new File(rootPath);
if (!rootPathFile.exists() || !rootPathFile.canWrite()) {
continue;
}
String logDirectory = rootPath + File.separator + "logs" + File.separator + "rocketmqlogs";
// Create directories recursively.
UtilAll.ensureDirOK(logDirectory);
return logDirectory;
}
rootPath = rootPath + File.separator + "logs";
UtilAll.ensureDirOK(rootPath);
return rootPath + File.separator + "rocketmqlogs" + File.separator;
throw new RuntimeException("Failed to get log directory");
}
}
common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java
+9 -1
View File
@@ -16,8 +16,11 @@
*/
package org.apache.rocketmq.common.message;
import com.google.common.base.Strings;
import java.nio.ByteBuffer;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicFilterType;
import org.apache.rocketmq.common.utils.MessageUtils;
@@ -41,7 +44,7 @@ public class MessageExtBrokerInner extends MessageExt {
}
public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) {
if (null == tags || tags.length() == 0) { return 0; }
if (Strings.isNullOrEmpty(tags)) { return 0; }
return tags.hashCode();
}
@@ -102,4 +105,9 @@ public class MessageExtBrokerInner extends MessageExt {
public void setEncodeCompleted(boolean encodeCompleted) {
this.encodeCompleted = encodeCompleted;
}
public boolean needDispatchLMQ() {
return StringUtils.isNoneBlank(getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH))
&& MixAll.topicAllowsLMQ(getTopic());
}
}
common/src/test/java/org/apache/rocketmq/common/config/ConfigHelperTest.java
+30
View File
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.common.config;
import org.junit.Test;
public class ConfigHelperTest {
@Test
public void testGetDBLogDir() {
// Should not raise exception.
ConfigHelper.getDBLogDir();
}
}
example/src/main/java/org/apache/rocketmq/example/lmq/LMQProducer.java
+1 -1
View File
@@ -49,7 +49,7 @@ public class LMQProducer {
Message msg = new Message(TOPIC, TAG, ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET));
msg.setKeys("Key" + i);
msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH /* "INNER_MULTI_DISPATCH" */,
String.join(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */);
String.join(MixAll.LMQ_DISPATCH_SEPARATOR, LMQ_TOPIC_1, LMQ_TOPIC_2) /* "%LMQ%123,%LMQ%456" */);
SendResult sendResult = producer.send(msg);
System.out.printf("%s%n", sendResult);
} catch (Exception e) {
remoting/src/test/java/org/apache/rocketmq/remoting/netty/FileRegionEncoderTest.java
+2 -2
View File
@@ -49,7 +49,7 @@ public class FileRegionEncoderTest {
random.nextBytes(data);
write(file, data);
FileRegion fileRegion = new DefaultFileRegion(file, 0, dataLength);
Assert.assertEquals(0, fileRegion.transfered());
Assert.assertEquals(0, fileRegion.transferred());
Assert.assertEquals(dataLength, fileRegion.count());
Assert.assertTrue(channel.writeOutbound(fileRegion));
ByteBuf out = (ByteBuf) channel.readOutbound();
@@ -77,4 +77,4 @@ public class FileRegionEncoderTest {
}
}
}
}
}
store/BUILD.bazel
+3
View File
@@ -42,6 +42,8 @@ java_library(
"@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
"@maven//:io_github_aliyunmq_rocketmq_logback_classic",
"@maven//:org_apache_rocketmq_rocketmq_rocksdb",
"@maven//:com_google_code_findbugs_jsr305",
"@maven//:commons_validator_commons_validator",
],
)
@@ -63,6 +65,7 @@ java_library(
"@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
"@maven//:io_github_aliyunmq_rocketmq_logback_classic",
"@maven//:org_apache_rocketmq_rocketmq_rocksdb",
"@maven//:org_junit_jupiter_junit_jupiter_api",
],
)
store/src/main/java/org/apache/rocketmq/store/AppendMessageStatus.java
+1
View File
@@ -25,4 +25,5 @@ public enum AppendMessageStatus {
MESSAGE_SIZE_EXCEEDED,
PROPERTIES_SIZE_EXCEEDED,
UNKNOWN_ERROR,
ROCKSDB_ERROR,
}
store/src/main/java/org/apache/rocketmq/store/CommitLog.java
+28 -18
View File
@@ -16,6 +16,7 @@
*/
package org.apache.rocketmq.store;
import com.google.common.base.Strings;
import java.net.Inet6Address;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
@@ -35,7 +36,6 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.sun.jna.NativeLong;
import com.sun.jna.Pointer;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.ServiceThread;
import org.apache.rocketmq.common.SystemClock;
@@ -58,10 +58,11 @@ import org.apache.rocketmq.store.MessageExtEncoder.PutMessageThreadLocal;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.exception.StoreException;
import org.apache.rocketmq.store.ha.HAService;
import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;
import org.apache.rocketmq.store.logfile.MappedFile;
import org.apache.rocketmq.store.queue.MultiDispatchUtils;
import org.apache.rocketmq.store.util.LibC;
import org.rocksdb.RocksDBException;
@@ -104,7 +105,6 @@ public class CommitLog implements Swappable {
protected int commitLogSize;
private final boolean enabledAppendPropCRC;
protected final MultiDispatch multiDispatch;
public CommitLog(final DefaultMessageStore messageStore) {
String storePath = messageStore.getMessageStoreConfig().getStorePathCommitLog();
@@ -139,8 +139,6 @@ public class CommitLog implements Swappable {
this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog();
this.enabledAppendPropCRC = messageStore.getMessageStoreConfig().isEnabledAppendPropCRC();
this.multiDispatch = new MultiDispatch(defaultMessageStore);
}
public void setFullStorePaths(Set<String> fullStorePaths) {
@@ -530,7 +528,7 @@ public class CommitLog implements Swappable {
}
String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS);
if (tags != null && tags.length() > 0) {
if (!Strings.isNullOrEmpty(tags)) {
tagsCode = MessageExtBrokerInner.tagsString2tagsCode(MessageExt.parseTopicFilterType(sysFlag), tags);
}
@@ -652,7 +650,7 @@ public class CommitLog implements Swappable {
} else if (this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable()) {
return this.confirmOffset;
} else {
return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset();
return this.defaultMessageStore.isSyncDiskFlush() ? getFlushedWhere() : getMaxOffset();
}
}
@@ -770,8 +768,11 @@ public class CommitLog implements Swappable {
}
}
// only for rocksdb mode
this.getMessageStore().finishCommitLogDispatch();
try {
this.getMessageStore().getQueueStore().flush();
} catch (StoreException e) {
log.error("Failed to flush ConsumeQueueStore", e);
}
processOffset += mappedFileOffset;
if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) {
@@ -988,7 +989,7 @@ public class CommitLog implements Swappable {
msg.setEncodedBuff(putMessageThreadLocal.getEncoder().getEncoderBuffer());
PutMessageContext putMessageContext = new PutMessageContext(topicQueueKey);
putMessageLock.lock(); //spin or ReentrantLock ,depending on store config
putMessageLock.lock(); //spin or ReentrantLock, depending on store config
try {
long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now();
this.beginTimeInLock = beginLockTimestamp;
@@ -1850,7 +1851,16 @@ public class CommitLog implements Swappable {
return null;
}
multiDispatch.wrapMultiDispatch(msgInner);
try {
LmqDispatch.wrapLmqDispatch(defaultMessageStore, msgInner);
} catch (ConsumeQueueException e) {
if (e.getCause() instanceof RocksDBException) {
log.error("Failed to wrap multi-dispatch", e);
return new AppendMessageResult(AppendMessageStatus.ROCKSDB_ERROR);
}
log.error("Failed to wrap multi-dispatch", e);
return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
}
msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties()));
@@ -1904,7 +1914,7 @@ public class CommitLog implements Swappable {
// STORETIMESTAMP + STOREHOSTADDRESS + OFFSET <br>
ByteBuffer preEncodeBuffer = msgInner.getEncodedBuff();
final boolean isMultiDispatchMsg = CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner);
boolean isMultiDispatchMsg = messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ();
if (isMultiDispatchMsg) {
AppendMessageResult appendMessageResult = handlePropertiesForLmqMsg(preEncodeBuffer, msgInner);
if (appendMessageResult != null) {
@@ -2000,7 +2010,12 @@ public class CommitLog implements Swappable {
msgInner.setEncodedBuff(null);
if (isMultiDispatchMsg) {
CommitLog.this.multiDispatch.updateMultiQueueOffset(msgInner);
try {
LmqDispatch.updateLmqOffsets(defaultMessageStore, msgInner);
} catch (ConsumeQueueException e) {
// Increase in-memory max offset of the queue should not fail.
return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR);
}
}
return new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgIdSupplier,
@@ -2245,11 +2260,6 @@ public class CommitLog implements Swappable {
return flushManager;
}
public static boolean isMultiDispatchMsg(MessageStoreConfig messageStoreConfig, MessageExtBrokerInner msg) {
return StringUtils.isNotBlank(msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH)) &&
MultiDispatchUtils.isNeedHandleMultiDispatch(messageStoreConfig, msg.getTopic());
}
private boolean isCloseReadAhead() {
return !MixAll.isWindows() && !defaultMessageStore.getMessageStoreConfig().isDataReadAheadEnable();
}
store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java
+2 -2
View File
@@ -727,8 +727,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle {
Map<String, String> prop = request.getPropertiesMap();
String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);
String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);
if (queues.length != queueOffsets.length) {
log.error("[bug] queues.length!=queueOffsets.length ", request.getTopic());
return;
store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java
+29 -37
View File
@@ -93,6 +93,7 @@ import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.dledger.DLedgerCommitLog;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.ha.DefaultHAService;
import org.apache.rocketmq.store.ha.HAService;
import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService;
@@ -170,7 +171,7 @@ public class DefaultMessageStore implements MessageStore {
private RocksDBMessageStore rocksDBMessageStore;
private RandomAccessFile lockFile;
private final RandomAccessFile lockFile;
private FileLock lock;
@@ -190,7 +191,7 @@ public class DefaultMessageStore implements MessageStore {
private volatile long brokerInitMaxOffset = -1L;
private List<PutMessageHook> putMessageHookList = new ArrayList<>();
private final List<PutMessageHook> putMessageHookList = new ArrayList<>();
private SendMessageBackHook sendMessageBackHook;
@@ -203,20 +204,21 @@ public class DefaultMessageStore implements MessageStore {
private final ConcurrentLinkedQueue<BatchDispatchRequest> batchDispatchRequestQueue = new ConcurrentLinkedQueue<>();
private int dispatchRequestOrderlyQueueSize = 16;
private final int dispatchRequestOrderlyQueueSize = 16;
private final DispatchRequestOrderlyQueue dispatchRequestOrderlyQueue = new DispatchRequestOrderlyQueue(dispatchRequestOrderlyQueueSize);
private long stateMachineVersion = 0L;
// this is a unmodifiableMap
private ConcurrentMap<String, TopicConfig> topicConfigTable;
private final ConcurrentMap<String, TopicConfig> topicConfigTable;
private final ScheduledExecutorService scheduledCleanQueueExecutorService =
ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread"));
public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager,
final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {
final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig,
final ConcurrentMap<String, TopicConfig> topicConfigTable) throws IOException {
this.messageArrivingListener = messageArrivingListener;
this.brokerConfig = brokerConfig;
this.messageStoreConfig = messageStoreConfig;
@@ -438,7 +440,7 @@ public class DefaultMessageStore implements MessageStore {
return;
}
/**
/*
* 1. Make sure the fast-forward messages to be truncated during the recovering according to the max physical offset of the commitlog;
* 2. DLedger committedPos may be missing, so the maxPhysicalPosInLogicQueue maybe bigger that maxOffset returned by DLedgerCommitLog, just let it go;
* 3. Calculate the reput offset according to the consume queue;
@@ -458,7 +460,7 @@ public class DefaultMessageStore implements MessageStore {
}
if (maxPhysicalPosInLogicQueue < this.commitLog.getMinOffset()) {
maxPhysicalPosInLogicQueue = this.commitLog.getMinOffset();
/**
/*
* This happens in following conditions:
* 1. If someone removes all the consumequeue files or the disk get damaged.
* 2. Launch a new broker, and copy the commitlog from other brokers.
@@ -987,12 +989,12 @@ public class DefaultMessageStore implements MessageStore {
}
@Override
public long getMaxOffsetInQueue(String topic, int queueId) {
public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {
return getMaxOffsetInQueue(topic, queueId, true);
}
@Override
public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) {
public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {
if (committed) {
ConsumeQueueInterface logic = this.getConsumeQueue(topic, queueId);
if (logic != null) {
@@ -1378,7 +1380,6 @@ public class DefaultMessageStore implements MessageStore {
* If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned,
* consume queue will be created with old offset, then later message with new offset table can not be
* dispatched to consume queue.
* @throws RocksDBException only in rocksdb mode
*/
@Override
public int deleteTopics(final Set<String> deleteTopics) {
@@ -1748,10 +1749,11 @@ public class DefaultMessageStore implements MessageStore {
/**
* The ratio val is estimated by the experiment and experience
* so that the result is not high accurate for different business
*
* @return
*/
public boolean checkInColdAreaByCommitOffset(long offsetPy, long maxOffsetPy) {
long memory = (long)(StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0));
long memory = (long) (StoreUtil.TOTAL_PHYSICAL_MEMORY_SIZE * (this.messageStoreConfig.getAccessMessageInMemoryHotRatio() / 100.0));
return (maxOffsetPy - offsetPy) > memory;
}
@@ -1929,11 +1931,6 @@ public class DefaultMessageStore implements MessageStore {
return messageStoreConfig;
}
@Override
public void finishCommitLogDispatch() {
// ignore
}
@Override
public TransientStorePool getTransientStorePool() {
return transientStorePool;
@@ -2713,15 +2710,15 @@ public class DefaultMessageStore implements MessageStore {
}
}
class BatchDispatchRequest {
static class BatchDispatchRequest {
private ByteBuffer byteBuffer;
private final ByteBuffer byteBuffer;
private int position;
private final int position;
private int size;
private final int size;
private long id;
private final long id;
public BatchDispatchRequest(ByteBuffer byteBuffer, int position, int size, long id) {
this.byteBuffer = byteBuffer;
@@ -2731,7 +2728,7 @@ public class DefaultMessageStore implements MessageStore {
}
}
class DispatchRequestOrderlyQueue {
static class DispatchRequestOrderlyQueue {
DispatchRequest[][] buffer;
@@ -2907,8 +2904,6 @@ public class DefaultMessageStore implements MessageStore {
} finally {
result.release();
}
finishCommitLogDispatch();
}
}
@@ -2922,8 +2917,8 @@ public class DefaultMessageStore implements MessageStore {
if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) {
return;
}
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
String[] queueOffsets = multiQueueOffset.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
String[] queues = multiDispatchQueue.split(MixAll.LMQ_DISPATCH_SEPARATOR);
String[] queueOffsets = multiQueueOffset.split(MixAll.LMQ_DISPATCH_SEPARATOR);
if (queues.length != queueOffsets.length) {
return;
}
@@ -2932,7 +2927,7 @@ public class DefaultMessageStore implements MessageStore {
long queueOffset = Long.parseLong(queueOffsets[i]);
int queueId = dispatchRequest.getQueueId();
if (DefaultMessageStore.this.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
queueId = 0;
queueId = MixAll.LMQ_QUEUE_ID;
}
DefaultMessageStore.this.messageArrivingListener.arriving(
queueName, queueId, queueOffset + 1, dispatchRequest.getTagsCode(),
@@ -2972,13 +2967,13 @@ public class DefaultMessageStore implements MessageStore {
public MainBatchDispatchRequestService() {
batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor(
DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
1000 * 60,
TimeUnit.MICROSECONDS,
new LinkedBlockingQueue<>(4096),
new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
new ThreadPoolExecutor.AbortPolicy());
DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(),
1000 * 60,
TimeUnit.MICROSECONDS,
new LinkedBlockingQueue<>(4096),
new ThreadFactoryImpl("BatchDispatchRequestServiceThread_"),
new ThreadPoolExecutor.AbortPolicy());
}
private void pollBatchDispatchRequest() {
@@ -3188,9 +3183,6 @@ public class DefaultMessageStore implements MessageStore {
result.release();
}
}
// only for rocksdb mode
finishCommitLogDispatch();
}
/**
store/src/main/java/org/apache/rocketmq/store/DispatchRequest.java
+15
View File
@@ -17,6 +17,9 @@
package org.apache.rocketmq.store;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageConst;
public class DispatchRequest {
private final String topic;
@@ -228,6 +231,18 @@ public class DispatchRequest {
this.offsetId = offsetId;
}
public boolean containsLMQ() {
if (!MixAll.topicAllowsLMQ(topic)) {
return false;
}
if (null == propertiesMap || propertiesMap.isEmpty()) {
return false;
}
String lmqNames = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String lmqOffsets = propertiesMap.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
return !StringUtils.isBlank(lmqNames) && !StringUtils.isBlank(lmqOffsets);
}
@Override
public String toString() {
return "DispatchRequest{" +
store/src/main/java/org/apache/rocketmq/store/LmqDispatch.java
+56
View File
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public class LmqDispatch {
private static final short VALUE_OF_EACH_INCREMENT = 1;
public static void wrapLmqDispatch(MessageStore messageStore, final MessageExtBrokerInner msg)
throws ConsumeQueueException {
String lmqNames = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
Long[] queueOffsets = new Long[queueNames.length];
if (messageStore.getMessageStoreConfig().isEnableLmq()) {
for (int i = 0; i < queueNames.length; i++) {
if (MixAll.isLmq(queueNames[i])) {
queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(queueNames[i], MixAll.LMQ_QUEUE_ID);
}
}
}
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
StringUtils.join(queueOffsets, MixAll.LMQ_DISPATCH_SEPARATOR));
msg.removeWaitStorePropertyString();
}
public static void updateLmqOffsets(MessageStore messageStore, final MessageExtBrokerInner msgInner)
throws ConsumeQueueException {
String lmqNames = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String[] queueNames = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
for (String queueName : queueNames) {
if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
messageStore.getQueueStore().increaseLmqOffset(queueName, MixAll.LMQ_QUEUE_ID, VALUE_OF_EACH_INCREMENT);
}
}
}
}
store/src/main/java/org/apache/rocketmq/store/MessageExtEncoder.java
+2 -2
View File
@@ -175,11 +175,11 @@ public class MessageExtEncoder {
public PutMessageResult encode(MessageExtBrokerInner msgInner) {
this.byteBuf.clear();
if (CommitLog.isMultiDispatchMsg(messageStoreConfig, msgInner)) {
if (messageStoreConfig.isEnableLmq() && msgInner.needDispatchLMQ()) {
return encodeWithoutProperties(msgInner);
}
/**
/*
* Serialize message
*/
final byte[] propertiesData =
store/src/main/java/org/apache/rocketmq/store/MessageStore.java
+5 -10
View File
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.function.Supplier;
import javax.annotation.Nonnull;
import org.apache.rocketmq.common.BoundaryType;
import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.SystemClock;
@@ -31,6 +32,7 @@ import org.apache.rocketmq.common.message.MessageExtBatch;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.ha.HAService;
import org.apache.rocketmq.store.hook.PutMessageHook;
import org.apache.rocketmq.store.hook.SendMessageBackHook;
@@ -181,7 +183,7 @@ public interface MessageStore {
* @param queueId Queue ID.
* @return Maximum offset at present.
*/
long getMaxOffsetInQueue(final String topic, final int queueId);
long getMaxOffsetInQueue(final String topic, final int queueId) throws ConsumeQueueException;
/**
* Get maximum offset of the topic queue.
@@ -191,7 +193,7 @@ public interface MessageStore {
* @param committed return the max offset in ConsumeQueue if true, or the max offset in CommitLog if false
* @return Maximum offset at present.
*/
long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed);
long getMaxOffsetInQueue(final String topic, final int queueId, final boolean committed) throws ConsumeQueueException;
/**
* Get the minimum offset of the topic queue.
@@ -626,14 +628,6 @@ public interface MessageStore {
void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile,
boolean isRecover, boolean isFileEnd) throws RocksDBException;
/**
* Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests)
* It will be triggered in two cases:
* @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput
* @see CommitLog#recoverAbnormally
*/
void finishCommitLogDispatch();
/**
* Get the message store config
*
@@ -724,6 +718,7 @@ public interface MessageStore {
*
* @return the queue store
*/
@Nonnull
ConsumeQueueStoreInterface getQueueStore();
/**
store/src/main/java/org/apache/rocketmq/store/MultiDispatch.java
-77
View File
@@ -1,77 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageAccessor;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
/**
* MultiDispatch for lmq, not-thread-safe
*/
public class MultiDispatch {
private final StringBuilder keyBuilder = new StringBuilder();
private final DefaultMessageStore messageStore;
private static final short VALUE_OF_EACH_INCREMENT = 1;
public MultiDispatch(DefaultMessageStore messageStore) {
this.messageStore = messageStore;
}
public String queueKey(String queueName, MessageExtBrokerInner msgInner) {
keyBuilder.delete(0, keyBuilder.length());
keyBuilder.append(queueName);
keyBuilder.append('-');
int queueId = msgInner.getQueueId();
if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
queueId = 0;
}
keyBuilder.append(queueId);
return keyBuilder.toString();
}
public void wrapMultiDispatch(final MessageExtBrokerInner msg) {
String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
Long[] queueOffsets = new Long[queues.length];
if (messageStore.getMessageStoreConfig().isEnableLmq()) {
for (int i = 0; i < queues.length; i++) {
String key = queueKey(queues[i], msg);
if (MixAll.isLmq(key)) {
queueOffsets[i] = messageStore.getQueueStore().getLmqQueueOffset(key);
}
}
}
MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET,
StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER));
msg.removeWaitStorePropertyString();
}
public void updateMultiQueueOffset(final MessageExtBrokerInner msgInner) {
String multiDispatchQueue = msgInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER);
for (String queue : queues) {
String key = queueKey(queue, msgInner);
if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) {
messageStore.getQueueStore().increaseLmqOffset(key, VALUE_OF_EACH_INCREMENT);
}
}
}
}
store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java
-9
View File
@@ -79,15 +79,6 @@ public class RocksDBMessageStore extends DefaultMessageStore {
this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>());
}
@Override
public void finishCommitLogDispatch() {
try {
putMessagePositionInfo(null);
} catch (RocksDBException e) {
ERROR_LOG.info("try to finish commitlog dispatch error.", e);
}
}
@Override
public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) {
return findConsumeQueue(topic, queueId);
store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java
-10
View File
@@ -428,8 +428,6 @@ public class MessageStoreConfig {
private boolean rocksdbCQDoubleWriteEnable = false;
private int batchWriteKvCqSize = 16;
/**
* If ConsumeQueueStore is RocksDB based, this option is to configure bottom-most tier compression type.
* The following values are valid:
@@ -447,14 +445,6 @@ public class MessageStoreConfig {
*/
private String bottomMostCompressionTypeForConsumeQueueStore = "zstd";
public int getBatchWriteKvCqSize() {
return batchWriteKvCqSize;
}
public void setBatchWriteKvCqSize(int batchWriteKvCqSize) {
this.batchWriteKvCqSize = batchWriteKvCqSize;
}
public boolean isRocksdbCQDoubleWriteEnable() {
return rocksdbCQDoubleWriteEnable;
}
store/src/main/java/org/apache/rocketmq/store/exception/ConsumeQueueException.java
+39
View File
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.exception;
public class ConsumeQueueException extends StoreException {
public ConsumeQueueException() {
}
public ConsumeQueueException(String message) {
super(message);
}
public ConsumeQueueException(String message, Throwable cause) {
super(message, cause);
}
public ConsumeQueueException(Throwable cause) {
super(cause);
}
public ConsumeQueueException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
store/src/main/java/org/apache/rocketmq/store/exception/StoreException.java
+38
View File
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.exception;
public class StoreException extends Exception {
public StoreException() {
}
public StoreException(String message) {
super(message);
}
public StoreException(String message, Throwable cause) {
super(message, cause);
}
public StoreException(Throwable cause) {
super(cause);
}
public StoreException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java
+4 -8
View File
@@ -46,6 +46,7 @@ import org.apache.rocketmq.store.StoreCheckpoint;
import org.apache.rocketmq.store.StoreStatsService;
import org.apache.rocketmq.store.TransientStorePool;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.ha.HAService;
import org.apache.rocketmq.store.hook.PutMessageHook;
import org.apache.rocketmq.store.hook.SendMessageBackHook;
@@ -63,7 +64,7 @@ import io.opentelemetry.sdk.metrics.InstrumentSelector;
import io.opentelemetry.sdk.metrics.ViewBuilder;
public abstract class AbstractPluginMessageStore implements MessageStore {
protected MessageStore next = null;
protected MessageStore next;
protected MessageStorePluginContext context;
public AbstractPluginMessageStore(MessageStorePluginContext context, MessageStore next) {
@@ -139,12 +140,12 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
}
@Override
public long getMaxOffsetInQueue(String topic, int queueId) {
public long getMaxOffsetInQueue(String topic, int queueId) throws ConsumeQueueException {
return next.getMaxOffsetInQueue(topic, queueId);
}
@Override
public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) {
public long getMaxOffsetInQueue(String topic, int queueId, boolean committed) throws ConsumeQueueException {
return next.getMaxOffsetInQueue(topic, queueId, committed);
}
@@ -647,11 +648,6 @@ public abstract class AbstractPluginMessageStore implements MessageStore {
next.initMetrics(meter, attributesBuilderSupplier);
}
@Override
public void finishCommitLogDispatch() {
next.finishCommitLogDispatch();
}
@Override
public void recoverTopicQueueTable() {
next.recoverTopicQueueTable();
store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java
+9 -8
View File
@@ -25,6 +25,7 @@ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.rocksdb.RocksDBException;
public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface {
@@ -47,7 +48,7 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
}
@Override
public Long getMaxOffset(String topic, int queueId) {
public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {
return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId);
}
@@ -58,7 +59,7 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
}
@Override
public ConcurrentMap getTopicQueueTable() {
public ConcurrentMap<String, Long> getTopicQueueTable() {
return this.queueOffsetOperator.getTopicQueueTable();
}
@@ -75,13 +76,13 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
}
@Override
public void increaseLmqOffset(String queueKey, short messageNum) {
queueOffsetOperator.increaseLmqOffset(queueKey, messageNum);
public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {
queueOffsetOperator.increaseLmqOffset(topic, queueId, delta);
}
@Override
public long getLmqQueueOffset(String queueKey) {
return queueOffsetOperator.getLmqOffset(queueKey);
public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {
return queueOffsetOperator.getLmqOffset(topic, queueId, (t, q) -> 0L);
}
@Override
@@ -105,9 +106,9 @@ public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInte
try {
final long phyOffset = cqUnit.getPos();
final int size = cqUnit.getSize();
long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);
return storeTime;
return this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size);
} catch (Exception e) {
log.error("Failed to getStoreTime", e);
}
}
return -1;
store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java
+16
View File
@@ -47,6 +47,7 @@ import org.apache.rocketmq.store.ConsumeQueue;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.exception.StoreException;
import static java.lang.String.format;
import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue;
@@ -134,6 +135,12 @@ public class ConsumeQueueStore extends AbstractConsumeQueueStore {
@Override
public boolean shutdown() {
try {
flush();
} catch (StoreException e) {
log.error("Failed to flush all consume queues", e);
return false;
}
return true;
}
@@ -326,6 +333,15 @@ public class ConsumeQueueStore extends AbstractConsumeQueueStore {
return fileQueueLifeCycle.flush(flushLeastPages);
}
@Override
public void flush() throws StoreException {
for (Map.Entry<String, ConcurrentMap<Integer, ConsumeQueueInterface>> topicEntry : this.consumeQueueTable.entrySet()) {
for (Map.Entry<Integer, ConsumeQueueInterface> cqEntry : topicEntry.getValue().entrySet()) {
flush(cqEntry.getValue(), 0);
}
}
}
@Override
public void destroy(ConsumeQueueInterface consumeQueue) {
FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId());
store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java
+26 -13
View File
@@ -22,6 +22,8 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.rocketmq.common.BoundaryType;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.exception.StoreException;
import org.rocksdb.RocksDBException;
public interface ConsumeQueueStoreInterface {
@@ -79,10 +81,17 @@ public interface ConsumeQueueStoreInterface {
boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages);
/**
* clean expired data from minPhyOffset
* @param minPhyOffset
* Flush all nested consume queues to disk
*
* @throws StoreException if there is an error during flush
*/
void cleanExpired(long minPhyOffset);
void flush() throws StoreException;
/**
* clean expired data from minCommitLogOffset
* @param minCommitLogOffset Minimum commit log offset
*/
void cleanExpired(long minCommitLogOffset);
/**
* Check files.
@@ -92,10 +101,10 @@ public interface ConsumeQueueStoreInterface {
/**
* Delete expired files ending at min commit log position.
* @param consumeQueue
* @param minCommitLogPos min commit log position
* @param minCommitLogOffset min commit log position
* @return deleted file numbers.
*/
int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos);
int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogOffset);
/**
* Is the first file available?
@@ -185,17 +194,19 @@ public interface ConsumeQueueStoreInterface {
/**
* Increase lmq offset
* @param queueKey
* @param messageNum
* @param topic Topic/Queue name
* @param queueId Queue ID
* @param delta amount to increase
*/
void increaseLmqOffset(String queueKey, short messageNum);
void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException;
/**
* get lmq queue offset
* @param queueKey
* @param topic
* @param queueId
* @return
*/
long getLmqQueueOffset(String queueKey);
long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException;
/**
* recover topicQueue table by minPhyOffset
@@ -232,11 +243,13 @@ public interface ConsumeQueueStoreInterface {
/**
* get maxOffset of specific topic-queueId in topicQueue table
* @param topic
* @param queueId
*
* @param topic Topic name
* @param queueId Queue identifier
* @return the max offset in QueueOffsetOperator
* @throws ConsumeQueueException if there is an error while retrieving max consume queue offset
*/
Long getMaxOffset(String topic, int queueId);
Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException;
/**
* get max physic offset in consumeQueue
store/src/main/java/org/apache/rocketmq/store/queue/DispatchEntry.java
+47
View File
@@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.queue;
import java.nio.charset.StandardCharsets;
import javax.annotation.Nonnull;
import org.apache.rocketmq.store.DispatchRequest;
/**
* Use Record when Java 16 is available
*/
public class DispatchEntry {
public byte[] topic;
public int queueId;
public long queueOffset;
public long commitLogOffset;
public int messageSize;
public long tagCode;
public long storeTimestamp;
public static DispatchEntry from(@Nonnull DispatchRequest request) {
DispatchEntry entry = new DispatchEntry();
entry.topic = request.getTopic().getBytes(StandardCharsets.UTF_8);
entry.queueId = request.getQueueId();
entry.queueOffset = request.getConsumeQueueOffset();
entry.commitLogOffset = request.getCommitLogOffset();
entry.messageSize = request.getMsgSize();
entry.tagCode = request.getTagsCode();
entry.storeTimestamp = request.getStoreTimestamp();
return entry;
}
}
store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializer.java
+23
View File
@@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.queue;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
public interface OffsetInitializer {
long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException;
}
store/src/main/java/org/apache/rocketmq/store/queue/OffsetInitializerRocksDBImpl.java
+44
View File
@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.queue;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.rocksdb.RocksDBException;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
public class OffsetInitializerRocksDBImpl implements OffsetInitializer {
private static final Logger LOGGER = LoggerFactory.getLogger(OffsetInitializerRocksDBImpl.class);
private final RocksDBConsumeQueueStore consumeQueueStore;
public OffsetInitializerRocksDBImpl(RocksDBConsumeQueueStore consumeQueueStore) {
this.consumeQueueStore = consumeQueueStore;
}
@Override
public long maxConsumeQueueOffset(String topic, int queueId) throws ConsumeQueueException {
try {
long offset = consumeQueueStore.getMaxOffsetInQueue(topic, queueId);
LOGGER.info("Look up RocksDB for max-offset of LMQ[{}:{}]: {}", topic, queueId, offset);
return offset;
} catch (RocksDBException e) {
throw new ConsumeQueueException(e);
}
}
}
store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java
+28 -11
View File
@@ -17,6 +17,7 @@
package org.apache.rocketmq.store.queue;
import com.google.common.base.Preconditions;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -26,6 +27,7 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
/**
* QueueOffsetOperator is a component for operating offsets for queues.
@@ -35,7 +37,11 @@ public class QueueOffsetOperator {
private ConcurrentMap<String, Long> topicQueueTable = new ConcurrentHashMap<>(1024);
private ConcurrentMap<String, Long> batchTopicQueueTable = new ConcurrentHashMap<>(1024);
private ConcurrentMap<String/* topic-queueid */, Long/* offset */> lmqTopicQueueTable = new ConcurrentHashMap<>(1024);
/**
* {TOPIC}-{QUEUE_ID} --> NEXT Consume Queue Offset
*/
private ConcurrentMap<String/* topic-queue-id */, Long/* offset */> lmqTopicQueueTable = new ConcurrentHashMap<>(1024);
public long getQueueOffset(String topicQueueKey) {
return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L);
@@ -63,17 +69,28 @@ public class QueueOffsetOperator {
this.batchTopicQueueTable.put(topicQueueKey, batchQueueOffset + messageNum);
}
public long getLmqOffset(String topicQueueKey) {
return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L);
public long getLmqOffset(String topic, int queueId, OffsetInitializer callback) throws ConsumeQueueException {
Preconditions.checkNotNull(callback, "ConsumeQueueOffsetCallback cannot be null");
String topicQueue = topic + "-" + queueId;
if (!lmqTopicQueueTable.containsKey(topicQueue)) {
// Load from RocksDB on cache miss.
Long prev = lmqTopicQueueTable.putIfAbsent(topicQueue, callback.maxConsumeQueueOffset(topic, queueId));
if (null != prev) {
log.error("[BUG] Data racing, lmqTopicQueueTable should NOT contain key={}", topicQueue);
}
}
return lmqTopicQueueTable.get(topicQueue);
}
public Long getLmqTopicQueueNextOffset(String topicQueueKey) {
return this.lmqTopicQueueTable.get(topicQueueKey);
}
public void increaseLmqOffset(String queueKey, short messageNum) {
Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, queueKey, k -> 0L);
this.lmqTopicQueueTable.put(queueKey, lmqOffset + messageNum);
public void increaseLmqOffset(String topic, int queueId, short delta) throws ConsumeQueueException {
String topicQueue = topic + "-" + queueId;
if (!this.lmqTopicQueueTable.containsKey(topicQueue)) {
throw new ConsumeQueueException(String.format("Max offset of Queue[name=%s, id=%d] should have existed", topic, queueId));
}
long prev = lmqTopicQueueTable.get(topicQueue);
this.lmqTopicQueueTable.compute(topicQueue, (k, offset) -> offset + delta);
long current = lmqTopicQueueTable.get(topicQueue);
log.debug("Max offset of LMQ[{}:{}] increased: {} --> {}", topic, queueId, prev, current);
}
public long currentQueueOffset(String topicQueueKey) {
@@ -112,4 +129,4 @@ public class QueueOffsetOperator {
public void setBatchTopicQueueTable(ConcurrentMap<String, Long> batchTopicQueueTable) {
this.batchTopicQueueTable = batchTopicQueueTable;
}
}
}
store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java
+195 -93
View File
@@ -16,7 +16,10 @@
*/
package org.apache.rocketmq.store.queue;
import io.netty.util.internal.PlatformDependent;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -24,31 +27,33 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.topic.TopicValidator;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.ConsumeQueue;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.queue.offset.OffsetEntry;
import org.apache.rocketmq.store.queue.offset.OffsetEntryType;
import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import org.rocksdb.RocksIterator;
import org.rocksdb.WriteBatch;
import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;
public class RocksDBConsumeQueueOffsetTable {
private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME);
private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8);
private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8);
private static final byte[] MAX_BYTES = "max".getBytes(StandardCharsets.UTF_8);
private static final byte[] MIN_BYTES = "min".getBytes(StandardCharsets.UTF_8);
/**
* Rocksdb ConsumeQueue's Offset unit. Format:
@@ -72,10 +77,9 @@ public class RocksDBConsumeQueueOffsetTable {
* </pre>
* ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes
*/
private static final int OFFSET_PHY_OFFSET = 0;
private static final int OFFSET_CQ_OFFSET = 8;
static final int OFFSET_PHY_OFFSET = 0;
static final int OFFSET_CQ_OFFSET = 8;
/**
*
* ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐
* │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │
* │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │
@@ -86,16 +90,18 @@ public class RocksDBConsumeQueueOffsetTable {
/**
* We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread.
*
* @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread.
* If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of
* RocksDBConsumeQueueOffsetTable to find it.
*/
private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC;
private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8);
private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(StandardCharsets.UTF_8);
private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length;
private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN);
private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN];
private final ByteBuffer maxPhyOffsetBB;
static {
buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true);
INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN);
@@ -111,69 +117,147 @@ public class RocksDBConsumeQueueOffsetTable {
/**
* Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them
* from heap to avoid accessing rocksdb.
*
* @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset
* @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset
*/
private final Map<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;
private final Map<String/* topic-queueId */, Long> topicQueueMaxCqOffset;
private final ConcurrentMap<String/* topic-queueId */, PhyAndCQOffset> topicQueueMinOffset;
private final ConcurrentMap<String/* topic-queueId */, Long> topicQueueMaxCqOffset;
public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable,
ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) {
this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable;
this.rocksDBStorage = rocksDBStorage;
this.messageStore = messageStore;
this.topicQueueMinOffset = new ConcurrentHashMap(1024);
this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024);
this.topicQueueMinOffset = new ConcurrentHashMap<>(1024);
this.topicQueueMaxCqOffset = new ConcurrentHashMap<>(1024);
this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8);
}
public void load() {
this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle();
loadMaxConsumeQueueOffsets();
}
public void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
final byte[] topicBytes, final DispatchRequest request,
final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request);
ByteBuffer topicQueueId = offsetBBPair.getObject1();
ByteBuffer maxOffsetBB = offsetBBPair.getObject2();
Pair<ByteBuffer, DispatchRequest> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);
if (old == null) {
tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request));
} else {
long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET);
long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET);
if (maxOffset >= oldMaxOffset) {
ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset);
}
private void loadMaxConsumeQueueOffsets() {
Function<OffsetEntry, Boolean> predicate = entry -> entry.type == OffsetEntryType.MAXIMUM;
Consumer<OffsetEntry> fn = entry -> {
topicQueueMaxCqOffset.putIfAbsent(entry.topic + "-" + entry.queueId, entry.offset);
ROCKSDB_LOG.info("Max {}:{} --> {}|{}", entry.topic, entry.queueId, entry.offset, entry.commitLogOffset);
};
try {
forEach(predicate, fn);
} catch (RocksDBException e) {
log.error("Failed to maximum consume queue offset", e);
}
}
public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap,
public void forEach(Function<OffsetEntry, Boolean> predicate, Consumer<OffsetEntry> fn) throws RocksDBException {
try (RocksIterator iterator = this.rocksDBStorage.seekOffsetCF()) {
if (null == iterator) {
return;
}
int keyBufferCapacity = 256;
iterator.seekToFirst();
ByteBuffer keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);
ByteBuffer valueBuffer = ByteBuffer.allocateDirect(16);
while (iterator.isValid()) {
// parse key buffer according to key layout
keyBuffer.clear(); // clear position and limit before reuse
int total = iterator.key(keyBuffer);
if (total > keyBufferCapacity) {
keyBufferCapacity = total;
PlatformDependent.freeDirectBuffer(keyBuffer);
keyBuffer = ByteBuffer.allocateDirect(keyBufferCapacity);
continue;
}
if (keyBuffer.remaining() <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES) {
iterator.next();
ROCKSDB_LOG.warn("Malformed Key/Value pair");
continue;
}
int topicLength = keyBuffer.getInt();
byte ctrl1 = keyBuffer.get();
assert ctrl1 == CTRL_1;
byte[] topicBytes = new byte[topicLength];
keyBuffer.get(topicBytes);
ctrl1 = keyBuffer.get();
assert ctrl1 == CTRL_1;
String topic = new String(topicBytes, StandardCharsets.UTF_8);
byte[] minMax = new byte[3];
keyBuffer.get(minMax);
OffsetEntryType entryType;
if (Arrays.equals(minMax, MAX_BYTES)) {
entryType = OffsetEntryType.MAXIMUM;
} else {
entryType = OffsetEntryType.MINIMUM;
}
ctrl1 = keyBuffer.get();
assert ctrl1 == CTRL_1;
assert keyBuffer.remaining() == Integer.BYTES;
int queueId = keyBuffer.getInt();
// Read and parse value buffer according to value layout
valueBuffer.clear(); // clear position and limit before reuse
total = iterator.value(valueBuffer);
if (total != Long.BYTES + Long.BYTES) {
// Skip system checkpoint topic as its value is only 8 bytes
iterator.next();
continue;
}
long commitLogOffset = valueBuffer.getLong();
long consumeOffset = valueBuffer.getLong();
OffsetEntry entry = new OffsetEntry();
entry.topic = topic;
entry.queueId = queueId;
entry.type = entryType;
entry.offset = consumeOffset;
entry.commitLogOffset = commitLogOffset;
if (predicate.apply(entry)) {
fn.accept(entry);
}
iterator.next();
}
// clean up direct buffers
PlatformDependent.freeDirectBuffer(keyBuffer);
PlatformDependent.freeDirectBuffer(valueBuffer);
}
}
public void putMaxPhyAndCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap,
final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException {
for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1());
}
appendMaxPhyOffset(writeBatch, maxPhyOffset);
}
public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap) {
for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
DispatchRequest request = entry.getValue().getObject2();
putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset());
public void putHeapMaxCqOffset(final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap) {
for (Map.Entry<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> entry : tempTopicQueueMaxOffsetMap.entrySet()) {
DispatchEntry dispatchEntry = entry.getValue().getObject2();
String topic = new String(dispatchEntry.topic, StandardCharsets.UTF_8);
putHeapMaxCqOffset(topic, dispatchEntry.queueId, dispatchEntry.queueOffset);
}
}
/**
* When topic is deleted, we clean up its offset info in rocksdb.
*
* @param topic
* @param queueId
* @throws RocksDBException
*/
public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false);
byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array());
Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null;
@@ -214,15 +298,14 @@ public class RocksDBConsumeQueueOffsetTable {
/**
* Traverse the offset table to find dirty topic
*
* @param existTopicSet
* @return
*/
public Map<String, Set<Integer>> iterateOffsetTable2FindDirty(final Set<String> existTopicSet) {
Map<String/* topic */, Set<Integer/* queueId */>> topicQueueIdToBeDeletedMap = new HashMap<>();
RocksIterator iterator = null;
try {
iterator = rocksDBStorage.seekOffsetCF();
try (RocksIterator iterator = rocksDBStorage.seekOffsetCF()) {
if (iterator == null) {
return topicQueueIdToBeDeletedMap;
}
@@ -236,17 +319,22 @@ public class RocksDBConsumeQueueOffsetTable {
ByteBuffer keyBB = ByteBuffer.wrap(key);
int topicLen = keyBB.getInt(0);
byte[] topicBytes = new byte[topicLen];
/**
/*
* "Topic Bytes Array Size" + "CTRL_1" = 4 + 1
*/
keyBB.position(4 + 1);
keyBB.get(topicBytes);
String topic = new String(topicBytes, CHARSET_UTF8);
String topic = new String(topicBytes, StandardCharsets.UTF_8);
if (TopicValidator.isSystemTopic(topic)) {
continue;
}
/**
// LMQ topic offsets should NOT be removed
if (MixAll.isLmq(topic)) {
continue;
}
/*
* "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1"
* = 4 + 1 + topicLen + 1 + 3 + 1
*/
@@ -270,10 +358,6 @@ public class RocksDBConsumeQueueOffsetTable {
}
} catch (Exception e) {
ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e);
} finally {
if (iterator != null) {
iterator.close();
}
}
return topicQueueIdToBeDeletedMap;
}
@@ -285,9 +369,13 @@ public class RocksDBConsumeQueueOffsetTable {
final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId);
maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null;
String topicQueueId = buildTopicQueueId(topic, queueId);
this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L);
long offset = maxCqOffset != null ? maxCqOffset : -1L;
Long prev = this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, offset);
if (null == prev) {
ROCKSDB_LOG.info("Max offset of {} is initialized to {} according to RocksDB", topicQueueId, offset);
}
if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset);
ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, offset);
}
}
@@ -296,34 +384,50 @@ public class RocksDBConsumeQueueOffsetTable {
/**
* truncate dirty offset in rocksdb
*
* @param offsetToTruncate
* @throws RocksDBException
*/
public void truncateDirty(long offsetToTruncate) throws RocksDBException {
correctMaxPyhOffset(offsetToTruncate);
ConcurrentMap<String, TopicConfig> allTopicConfigMap = this.messageStore.getTopicConfigs();
if (allTopicConfigMap == null) {
return;
}
for (TopicConfig topicConfig : allTopicConfigMap.values()) {
for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) {
truncateDirtyOffset(topicConfig.getTopicName(), i);
Function<OffsetEntry, Boolean> predicate = entry -> {
if (entry.type == OffsetEntryType.MINIMUM) {
return false;
}
}
// Normal entry offset MUST have the following inequality
// entry commit-log offset + message-size-in-bytes <= offsetToTruncate;
// otherwise, the consume queue contains dirty records to truncate;
//
// If the broker node is configured to use async-flush, it's possible consume queues contain
// pointers to message records that is not flushed and lost during restart.
return entry.commitLogOffset >= offsetToTruncate;
};
Consumer<OffsetEntry> fn = entry -> {
try {
truncateDirtyOffset(entry.topic, entry.queueId);
} catch (RocksDBException e) {
log.error("Failed to truncate maximum offset of consume queue[topic={}, queue-id={}]",
entry.topic, entry.queueId, e);
}
};
forEach(predicate, fn);
}
private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException {
private Pair<Boolean, Long> isMinOffsetOk(final String topic, final int queueId,
final long minPhyOffset) throws RocksDBException {
PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId);
if (phyAndCQOffset != null) {
final long phyOffset = phyAndCQOffset.getPhyOffset();
final long cqOffset = phyAndCQOffset.getCqOffset();
return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset);
return (phyOffset >= minPhyOffset) ? new Pair<>(true, cqOffset) : new Pair<>(false, cqOffset);
}
ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId);
if (byteBuffer == null) {
return new Pair(false, 0L);
return new Pair<>(false, 0L);
}
final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET);
final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET);
@@ -334,9 +438,9 @@ public class RocksDBConsumeQueueOffsetTable {
if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset);
}
return new Pair(true, cqOffset);
return new Pair<>(true, cqOffset);
}
return new Pair(false, cqOffset);
return new Pair<>(false, cqOffset);
}
private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException {
@@ -361,8 +465,7 @@ public class RocksDBConsumeQueueOffsetTable {
if (!this.rocksDBStorage.hold()) {
return;
}
try {
WriteBatch writeBatch = new WriteBatch();
try (WriteBatch writeBatch = new WriteBatch()) {
long oldMaxPhyOffset = getMaxPhyOffset();
if (oldMaxPhyOffset <= maxPhyOffset) {
return;
@@ -416,10 +519,10 @@ public class RocksDBConsumeQueueOffsetTable {
}
private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
byte[] value = this.rocksDBStorage.getOffset(keyBB.array());
byte[] value = this.rocksDBStorage.getOffset(keyBB.array());
return (value != null) ? ByteBuffer.wrap(value) : null;
}
@@ -427,17 +530,19 @@ public class RocksDBConsumeQueueOffsetTable {
return topic + "-" + queueId;
}
private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) {
private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset,
final long minCQOffset) {
String topicQueueId = buildTopicQueueId(topic, queueId);
PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset);
this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset);
}
private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) {
private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxOffset) {
String topicQueueId = buildTopicQueueId(topic, queueId);
Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset);
if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) {
ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset);
Long prev = this.topicQueueMaxCqOffset.put(topicQueueId, maxOffset);
if (prev != null && prev > maxOffset) {
ERROR_LOG.error("Max offset of consume-queue[topic={}, queue-id={}] regressed. prev-max={}, current-max={}",
topic, queueId, prev, maxOffset);
}
}
@@ -463,9 +568,8 @@ public class RocksDBConsumeQueueOffsetTable {
if (!this.rocksDBStorage.hold()) {
return;
}
WriteBatch writeBatch = new WriteBatch();
try {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
try (WriteBatch writeBatch = new WriteBatch()) {
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max);
final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset);
@@ -481,7 +585,6 @@ public class RocksDBConsumeQueueOffsetTable {
ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e);
throw e;
} finally {
writeBatch.close();
this.rocksDBStorage.release();
if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) {
ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}",
@@ -504,10 +607,8 @@ public class RocksDBConsumeQueueOffsetTable {
throw new RocksDBException("correctMaxCqOffset error");
}
long high = maxCQOffset;
long low = minCQOffset;
PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high,
low, maxPhyOffsetInCQ, false);
PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,
minCQOffset, maxPhyOffsetInCQ, false);
long targetCQOffset = targetPhyAndCQOffset.getCqOffset();
long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset();
@@ -541,10 +642,8 @@ public class RocksDBConsumeQueueOffsetTable {
return true;
}
long high = maxCQOffset;
long low = minCQOffset;
PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low,
minPhyOffset, true);
PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, maxCQOffset,
minCQOffset, minPhyOffset, true);
long targetCQOffset = phyAndCQOffset.getCqOffset();
long targetPhyOffset = phyAndCQOffset.getPhyOffset();
@@ -568,28 +667,29 @@ public class RocksDBConsumeQueueOffsetTable {
return new Pair<>(offsetKey, offsetValue);
}
private void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
final byte[] topicBytes, final DispatchRequest request) {
static void buildOffsetKeyAndValueByteBuffer(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
final DispatchEntry entry) {
final ByteBuffer offsetKey = offsetBBPair.getObject1();
buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true);
buildOffsetKeyByteBuffer(offsetKey, entry.topic, entry.queueId, true);
final ByteBuffer offsetValue = offsetBBPair.getObject2();
buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset());
buildOffsetValueByteBuffer(offsetValue, entry.commitLogOffset, entry.queueOffset);
}
private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {
private static ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) {
ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
return byteBuffer;
}
private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) {
private static void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes,
final int queueId, final boolean max) {
byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length);
buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max);
}
private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId,
final boolean max) {
private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes,
final int queueId, final boolean max) {
byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1);
if (max) {
byteBuffer.put(MAX_BYTES);
@@ -600,18 +700,20 @@ public class RocksDBConsumeQueueOffsetTable {
byteBuffer.flip();
}
private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
private static void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset,
final long cqOffset) {
byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH);
buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
}
private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {
private static ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) {
final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH);
buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset);
return byteBuffer;
}
private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) {
private static void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset,
final long cqOffset) {
byteBuffer.putLong(phyOffset).putLong(cqOffset);
byteBuffer.flip();
}
store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java
+155 -67
View File
@@ -18,6 +18,7 @@ package org.apache.rocketmq.store.queue;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -28,21 +29,27 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.BoundaryType;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.Pair;
import org.apache.rocketmq.common.ThreadFactoryImpl;
import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.common.utils.DataConverter;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.exception.StoreException;
import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
import org.rocksdb.FlushOptions;
import org.rocksdb.RocksDBException;
import org.rocksdb.Statistics;
import org.rocksdb.WriteBatch;
@@ -51,11 +58,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME);
private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME);
public static final byte CTRL_0 = '\u0000';
public static final byte CTRL_1 = '\u0001';
public static final byte CTRL_2 = '\u0002';
private static final int DEFAULT_BYTE_BUFFER_CAPACITY = 16;
private final int batchSize;
public static final int MAX_KEY_LEN = 300;
private final ScheduledExecutorService scheduledExecutorService;
@@ -70,13 +74,16 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable;
private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable;
private final WriteBatch writeBatch;
private final List<DispatchRequest> bufferDRList;
private final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList;
private final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList;
private final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap;
private final Map<ByteBuffer, Pair<ByteBuffer, DispatchEntry>> tempTopicQueueMaxOffsetMap;
private volatile boolean isCQError = false;
private int consumeQueueByteBufferCacheIndex;
private int offsetBufferCacheIndex;
private final OffsetInitializer offsetInitializer;
public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) {
super(messageStore);
@@ -85,12 +92,10 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore);
this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore);
this.writeBatch = new WriteBatch();
this.batchSize = messageStoreConfig.getBatchWriteKvCqSize();
this.bufferDRList = new ArrayList<>(batchSize);
this.cqBBPairList = new ArrayList<>(batchSize);
this.offsetBBPairList = new ArrayList<>(batchSize);
for (int i = 0; i < batchSize; i++) {
this.offsetInitializer = new OffsetInitializerRocksDBImpl(this);
this.cqBBPairList = new ArrayList<>(16);
this.offsetBBPairList = new ArrayList<>(DEFAULT_BYTE_BUFFER_CAPACITY);
for (int i = 0; i < DEFAULT_BYTE_BUFFER_CAPACITY; i++) {
this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());
this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());
}
@@ -100,6 +105,22 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity()));
}
private Pair<ByteBuffer, ByteBuffer> getCQByteBufferPair() {
int idx = consumeQueueByteBufferCacheIndex++;
if (idx >= cqBBPairList.size()) {
this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair());
}
return cqBBPairList.get(idx);
}
private Pair<ByteBuffer, ByteBuffer> getOffsetByteBufferPair() {
int idx = offsetBufferCacheIndex++;
if (idx >= offsetBBPairList.size()) {
this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair());
}
return offsetBBPairList.get(idx);
}
@Override
public void start() {
log.info("RocksDB ConsumeQueueStore start!");
@@ -164,19 +185,19 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
@Override
public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException {
if (request == null || this.bufferDRList.size() >= batchSize) {
putMessagePosition();
}
if (request != null) {
this.bufferDRList.add(request);
if (null == request) {
return;
}
// We are taking advantage of Atomic Flush, this operation is purely memory-based.
// batch and cache in Java heap does not make sense, instead, we should put the metadata into RocksDB immediately
// to optimized overall end-to-end latency.
putMessagePosition(request);
}
public void putMessagePosition() throws RocksDBException {
public void putMessagePosition(DispatchRequest request) throws RocksDBException {
final int maxRetries = 30;
for (int i = 0; i < maxRetries; i++) {
if (putMessagePosition0()) {
if (putMessagePosition0(request)) {
if (this.isCQError) {
this.messageStore.getRunningFlags().clearLogicsQueueError();
this.isCQError = false;
@@ -198,81 +219,113 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
throw new RocksDBException("put CQ Failed");
}
private boolean putMessagePosition0() {
private boolean putMessagePosition0(DispatchRequest request) {
if (!this.rocksDBStorage.hold()) {
return false;
}
final Map<ByteBuffer, Pair<ByteBuffer, DispatchRequest>> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap;
try {
final List<DispatchRequest> bufferDRList = this.bufferDRList;
final int size = bufferDRList.size();
if (size == 0) {
return true;
}
final List<Pair<ByteBuffer, ByteBuffer>> cqBBPairList = this.cqBBPairList;
final List<Pair<ByteBuffer, ByteBuffer>> offsetBBPairList = this.offsetBBPairList;
final WriteBatch writeBatch = this.writeBatch;
try (WriteBatch writeBatch = new WriteBatch()) {
long maxPhyOffset = 0;
for (int i = size - 1; i >= 0; i--) {
final DispatchRequest request = bufferDRList.get(i);
final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8);
DispatchEntry entry = DispatchEntry.from(request);
dispatch(entry, writeBatch);
dispatchLMQ(request, writeBatch);
this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch);
this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i),
topicBytes, request, tempTopicQueueMaxOffsetMap);
final int msgSize = request.getMsgSize();
final long phyOffset = request.getCommitLogOffset();
if (phyOffset + msgSize >= maxPhyOffset) {
maxPhyOffset = phyOffset + msgSize;
}
final int msgSize = request.getMsgSize();
final long phyOffset = request.getCommitLogOffset();
if (phyOffset + msgSize >= maxPhyOffset) {
maxPhyOffset = phyOffset + msgSize;
}
this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset);
// clear writeBatch in batchPut
this.rocksDBStorage.batchPut(writeBatch);
this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap);
long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp();
long storeTimeStamp = request.getStoreTimestamp();
if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE
|| this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) {
this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp);
}
this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp);
notifyMessageArriveAndClear();
notifyMessageArrival(request);
return true;
} catch (Exception e) {
ERROR_LOG.error("putMessagePosition0 Failed.", e);
ERROR_LOG.error("putMessagePosition0 failed.", e);
return false;
} finally {
tempTopicQueueMaxOffsetMap.clear();
consumeQueueByteBufferCacheIndex = 0;
offsetBufferCacheIndex = 0;
this.rocksDBStorage.release();
}
}
private void notifyMessageArriveAndClear() {
final List<DispatchRequest> bufferDRList = this.bufferDRList;
try {
for (DispatchRequest dp : bufferDRList) {
this.messageStore.notifyMessageArriveIfNecessary(dp);
private void dispatch(@Nonnull DispatchEntry entry, @Nonnull final WriteBatch writeBatch) throws RocksDBException {
this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(getCQByteBufferPair(), entry, writeBatch);
updateTempTopicQueueMaxOffset(getOffsetByteBufferPair(), entry);
}
private void updateTempTopicQueueMaxOffset(final Pair<ByteBuffer, ByteBuffer> offsetBBPair,
final DispatchEntry entry) {
RocksDBConsumeQueueOffsetTable.buildOffsetKeyAndValueByteBuffer(offsetBBPair, entry);
ByteBuffer topicQueueId = offsetBBPair.getObject1();
ByteBuffer maxOffsetBB = offsetBBPair.getObject2();
Pair<ByteBuffer, DispatchEntry> old = tempTopicQueueMaxOffsetMap.get(topicQueueId);
if (old == null) {
tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair<>(maxOffsetBB, entry));
} else {
long oldMaxOffset = old.getObject1().getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);
long maxOffset = maxOffsetBB.getLong(RocksDBConsumeQueueOffsetTable.OFFSET_CQ_OFFSET);
if (maxOffset >= oldMaxOffset) {
ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset);
}
}
}
private void dispatchLMQ(@Nonnull DispatchRequest request, @Nonnull final WriteBatch writeBatch)
throws RocksDBException {
if (!messageStoreConfig.isEnableLmq() || !request.containsLMQ()) {
return;
}
Map<String, String> map = request.getPropertiesMap();
String lmqNames = map.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH);
String lmqOffsets = map.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET);
String[] queues = lmqNames.split(MixAll.LMQ_DISPATCH_SEPARATOR);
String[] queueOffsets = lmqOffsets.split(MixAll.LMQ_DISPATCH_SEPARATOR);
if (queues.length != queueOffsets.length) {
ERROR_LOG.error("[bug] queues.length!=queueOffsets.length ", request.getTopic());
return;
}
for (int i = 0; i < queues.length; i++) {
String queueName = queues[i];
DispatchEntry entry = DispatchEntry.from(request);
long queueOffset = Long.parseLong(queueOffsets[i]);
int queueId = request.getQueueId();
if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) {
queueId = MixAll.LMQ_QUEUE_ID;
}
entry.queueId = queueId;
entry.queueOffset = queueOffset;
entry.topic = queueName.getBytes(StandardCharsets.UTF_8);
log.debug("Dispatch LMQ[{}:{}]:{} --> {}", queueName, queueId, queueOffset, entry.commitLogOffset);
dispatch(entry, writeBatch);
}
}
private void notifyMessageArrival(DispatchRequest request) {
try {
this.messageStore.notifyMessageArriveIfNecessary(request);
} catch (Exception e) {
ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e);
} finally {
bufferDRList.clear();
}
}
public Statistics getStatistics() {
return rocksDBStorage.getStatistics();
}
@Override
public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex,
final int num) throws RocksDBException {
return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num);
}
@@ -284,6 +337,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
/**
* Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them
* when we use them, we call it lazy correction.
*
* @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short)
* @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int)
*/
@@ -310,8 +364,7 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
return;
}
WriteBatch writeBatch = new WriteBatch();
try {
try (WriteBatch writeBatch = new WriteBatch()) {
this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch);
this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch);
@@ -320,7 +373,6 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e);
throw e;
} finally {
writeBatch.close();
this.rocksDBStorage.release();
}
}
@@ -330,10 +382,22 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
try {
this.rocksDBStorage.flushWAL();
} catch (Exception e) {
log.error("Failed to flush WAL", e);
}
return true;
}
@Override
public void flush() throws StoreException {
try (FlushOptions flushOptions = new FlushOptions()) {
flushOptions.setWaitForFlush(true);
flushOptions.setAllowWriteStall(true);
this.rocksDBStorage.flush(flushOptions);
} catch (RocksDBException e) {
throw new StoreException(e);
}
}
@Override
public void checkSelf() {
// ignored
@@ -350,8 +414,9 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
* will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted.
* But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in
* RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset).
* @param offsetToTruncate
* @throws RocksDBException
*
* @param offsetToTruncate CommitLog offset to truncate to
* @throws RocksDBException If there is any error.
*/
@Override
public void truncateDirty(long offsetToTruncate) throws RocksDBException {
@@ -369,7 +434,8 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
}
@Override
public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException {
public long getOffsetInQueueByTime(String topic, int queueId, long timestamp,
BoundaryType boundaryType) throws RocksDBException {
final long minPhysicOffset = this.messageStore.getMinPhyOffset();
long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId);
Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
@@ -380,6 +446,15 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
minPhysicOffset, boundaryType);
}
/**
* This method actually returns NEXT slot index to use, starting from 0. For example, if the queue is empty,
* it returns 0, pointing to the first slot of the 0-based queue;
*
* @param topic Topic name
* @param queueId Queue ID
* @return Index of the next slot to push into
* @throws RocksDBException if RocksDB fails to fulfill the request.
*/
@Override
public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException {
Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId);
@@ -444,4 +519,17 @@ public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore {
public long getTotalSize() {
return 0;
}
@Override
public long getLmqQueueOffset(String topic, int queueId) throws ConsumeQueueException {
return queueOffsetOperator.getLmqOffset(topic, queueId, offsetInitializer);
}
@Override
public Long getMaxOffset(String topic, int queueId) throws ConsumeQueueException {
if (MixAll.isLmq(topic)) {
return getLmqQueueOffset(topic, queueId);
}
return super.getMaxOffset(topic, queueId);
}
}
store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java
+15 -16
View File
@@ -17,6 +17,7 @@
package org.apache.rocketmq.store.queue;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
@@ -26,17 +27,15 @@ import org.apache.rocketmq.common.constant.LoggerName;
import org.apache.rocketmq.logging.org.slf4j.Logger;
import org.apache.rocketmq.logging.org.slf4j.LoggerFactory;
import org.apache.rocketmq.store.DefaultMessageStore;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset;
import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDBException;
import org.rocksdb.WriteBatch;
import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8;
import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0;
import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1;
import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2;
import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_0;
import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_1;
import static org.apache.rocketmq.common.config.AbstractRocksDBStorage.CTRL_2;
/**
* We use RocksDBConsumeQueueTable to store cqUnit.
@@ -105,30 +104,30 @@ public class RocksDBConsumeQueueTable {
this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle();
}
public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair,
final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException {
public void buildAndPutCQByteBuffer(final Pair<ByteBuffer, ByteBuffer> cqBBPair, final DispatchEntry request,
final WriteBatch writeBatch) throws RocksDBException {
final ByteBuffer cqKey = cqBBPair.getObject1();
buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset());
buildCQKeyByteBuffer(cqKey, request.topic, request.queueId, request.queueOffset);
final ByteBuffer cqValue = cqBBPair.getObject2();
buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp());
buildCQValueByteBuffer(cqValue, request.commitLogOffset, request.messageSize, request.tagCode, request.storeTimestamp);
writeBatch.put(this.defaultCFH, cqKey, cqValue);
}
public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset);
byte[] value = this.rocksDBStorage.getCQ(keyBB.array());
return (value != null) ? ByteBuffer.wrap(value) : null;
}
public List<ByteBuffer> rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
final List<ColumnFamilyHandle> defaultCFHList = new ArrayList(num);
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final List<ColumnFamilyHandle> defaultCFHList = new ArrayList<>(num);
final ByteBuffer[] resultList = new ByteBuffer[num];
final List<Integer> kvIndexList = new ArrayList(num);
final List<byte[]> kvKeyList = new ArrayList(num);
final List<Integer> kvIndexList = new ArrayList<>(num);
final List<byte[]> kvKeyList = new ArrayList<>(num);
for (int i = 0; i < num; i++) {
final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i);
kvIndexList.add(i);
@@ -153,7 +152,7 @@ public class RocksDBConsumeQueueTable {
}
final int resultSize = resultList.length;
List<ByteBuffer> bbValueList = new ArrayList(resultSize);
List<ByteBuffer> bbValueList = new ArrayList<>(resultSize);
for (int i = 0; i < resultSize; i++) {
ByteBuffer byteBuffer = resultList[i];
if (byteBuffer == null) {
@@ -171,7 +170,7 @@ public class RocksDBConsumeQueueTable {
* @throws RocksDBException
*/
public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException {
final byte[] topicBytes = topic.getBytes(CHARSET_UTF8);
final byte[] topicBytes = topic.getBytes(StandardCharsets.UTF_8);
final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId);
final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId);
store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntry.java
+45
View File
@@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.queue.offset;
public class OffsetEntry {
/**
* Topic identifier. For now, it's topic name directly. In the future, we should use fixed length topic identifier.
*/
public String topic;
/**
* Queue ID
*/
public int queueId;
/**
* Flag if the entry is for maximum or minimum
*/
public OffsetEntryType type;
/**
* Maximum or minimum consume-queue offset.
*/
public long offset;
/**
* Maximum or minimum commit-log offset.
*/
public long commitLogOffset;
}
store/src/main/java/org/apache/rocketmq/store/queue/offset/OffsetEntryType.java
+23
View File
@@ -0,0 +1,23 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store.queue.offset;
public enum OffsetEntryType {
MAXIMUM,
MINIMUM
}
store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java
+3 -2
View File
@@ -54,6 +54,7 @@ import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -374,7 +375,7 @@ public class DefaultMessageStoreTest {
}
@Test
public void testPutMessage_whenMessagePropertyIsTooLong() {
public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {
String topicName = "messagePropertyIsTooLongTest";
MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1);
assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED);
@@ -539,7 +540,7 @@ public class DefaultMessageStoreTest {
}
@Test
public void testMaxOffset() throws InterruptedException {
public void testMaxOffset() throws InterruptedException, ConsumeQueueException {
int firstBatchMessages = 3;
int queueId = 0;
messageBody = storeMessage.getBytes();
store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java
-105
View File
@@ -1,105 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.rocketmq.store;
import java.io.File;
import java.net.InetSocketAddress;
import java.nio.charset.Charset;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.rocketmq.common.BrokerConfig;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.message.MessageConst;
import org.apache.rocketmq.common.message.MessageDecoder;
import org.apache.rocketmq.common.message.MessageExtBrokerInner;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.queue.MultiDispatchUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.rocksdb.RocksDBException;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class MultiDispatchTest {
private MultiDispatch multiDispatch;
private DefaultMessageStore messageStore;
@Before
public void init() throws Exception {
MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8);
messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4);
messageStoreConfig.setMaxHashSlotNum(100);
messageStoreConfig.setMaxIndexNum(100 * 10);
messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1");
messageStoreConfig.setStorePathCommitLog(
System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1" + File.separator + "commitlog");
messageStoreConfig.setEnableLmq(true);
messageStoreConfig.setEnableMultiDispatch(true);
BrokerConfig brokerConfig = new BrokerConfig();
//too much reference
messageStore = new DefaultMessageStore(messageStoreConfig, null, null, brokerConfig, new ConcurrentHashMap<>());
multiDispatch = new MultiDispatch(messageStore);
}
@After
public void destroy() {
UtilAll.deleteFile(new File(System.getProperty("java.io.tmpdir") + File.separator + "unitteststore1"));
}
@Test
public void lmqQueueKey() {
MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class);
when(messageExtBrokerInner.getQueueId()).thenReturn(2);
String ret = MultiDispatchUtils.lmqQueueKey("%LMQ%lmq123");
assertEquals(ret, "%LMQ%lmq123-0");
}
@Test
public void wrapMultiDispatch() throws RocksDBException {
MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue();
multiDispatch.wrapMultiDispatch(messageExtBrokerInner);
assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0");
}
private MessageExtBrokerInner buildMessageMultiQueue() {
MessageExtBrokerInner msg = new MessageExtBrokerInner();
msg.setTopic("test");
msg.setTags("TAG1");
msg.setKeys("Hello");
msg.setBody("aaa".getBytes(Charset.forName("UTF-8")));
msg.setKeys(String.valueOf(System.currentTimeMillis()));
msg.setQueueId(0);
msg.setSysFlag(0);
msg.setBornTimestamp(System.currentTimeMillis());
msg.setStoreHost(new InetSocketAddress("127.0.0.1", 54270));
msg.setBornHost(new InetSocketAddress("127.0.0.1", 10911));
for (int i = 0; i < 1; i++) {
msg.putUserProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH, "%LMQ%123,%LMQ%456");
}
msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties()));
return msg;
}
}
store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java
+3 -2
View File
@@ -56,6 +56,7 @@ import org.apache.rocketmq.store.config.BrokerRole;
import org.apache.rocketmq.store.config.FlushDiskType;
import org.apache.rocketmq.store.config.MessageStoreConfig;
import org.apache.rocketmq.store.config.StorePathConfigHelper;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.store.stats.BrokerStatsManager;
@@ -434,7 +435,7 @@ public class RocksDBMessageStoreTest {
}
@Test
public void testPutMessage_whenMessagePropertyIsTooLong() {
public void testPutMessage_whenMessagePropertyIsTooLong() throws ConsumeQueueException {
if (notExecuted()) {
return;
}
@@ -603,7 +604,7 @@ public class RocksDBMessageStoreTest {
}
@Test
public void testMaxOffset() {
public void testMaxOffset() throws ConsumeQueueException {
if (notExecuted()) {
return;
}
test/src/test/java/org/apache/rocketmq/test/offset/LagCalculationIT.java
+3 -2
View File
@@ -36,6 +36,7 @@ import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper;
import org.apache.rocketmq.remoting.protocol.filter.FilterAPI;
import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData;
import org.apache.rocketmq.store.DefaultMessageFilter;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.test.base.BaseConf;
import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer;
import org.apache.rocketmq.test.client.rmq.RMQNormalProducer;
@@ -80,7 +81,7 @@ public class LagCalculationIT extends BaseConf {
shutdown();
}
private Pair<Long, Long> getLag(List<MessageQueue> mqs) {
private Pair<Long, Long> getLag(List<MessageQueue> mqs) throws ConsumeQueueException {
long lag = 0;
long pullLag = 0;
for (BrokerController controller : brokerControllerList) {
@@ -120,7 +121,7 @@ public class LagCalculationIT extends BaseConf {
}
@Test
public void testCalculateLag() {
public void testCalculateLag() throws ConsumeQueueException {
int msgSize = 10;
List<MessageQueue> mqs = producer.getMessageQueue();
MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);
tieredstore/src/main/java/org/apache/rocketmq/tieredstore/core/MessageStoreDispatcherImpl.java
+5
View File
@@ -34,6 +34,7 @@ import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.store.DispatchRequest;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.SelectMappedBufferResult;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.store.queue.ConsumeQueueInterface;
import org.apache.rocketmq.store.queue.CqUnit;
import org.apache.rocketmq.tieredstore.MessageStoreConfig;
@@ -259,6 +260,10 @@ public class MessageStoreDispatcherImpl extends ServiceThread implements Message
}
);
}
} catch (ConsumeQueueException e) {
CompletableFuture<Boolean> future = new CompletableFuture<>();
future.completeExceptionally(e);
return future;
} finally {
flatFile.getFileLock().unlock();
}
tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java
+49 -40
View File
@@ -40,6 +40,7 @@ import org.apache.rocketmq.common.metrics.NopLongCounter;
import org.apache.rocketmq.common.metrics.NopLongHistogram;
import org.apache.rocketmq.common.metrics.NopObservableLongGauge;
import org.apache.rocketmq.store.MessageStore;
import org.apache.rocketmq.store.exception.ConsumeQueueException;
import org.apache.rocketmq.tieredstore.MessageStoreConfig;
import org.apache.rocketmq.tieredstore.common.FileSegmentType;
import org.apache.rocketmq.tieredstore.core.MessageStoreFetcher;
@@ -177,26 +178,30 @@ public class TieredStoreMetricsManager {
.ofLongs()
.buildWithCallback(measurement -> {
for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) {
try {
MessageQueue mq = flatFile.getMessageQueue();
long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
continue;
MessageQueue mq = flatFile.getMessageQueue();
long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
continue;
}
Attributes commitLogAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
.build();
Attributes consumeQueueAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
.build();
measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes);
} catch (ConsumeQueueException e) {
// TODO: handle exception here
}
Attributes commitLogAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
.build();
Attributes consumeQueueAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
.build();
measurement.record(Math.max(maxOffset - flatFile.getConsumeQueueMaxOffset(), 0), consumeQueueAttributes);
}
});
@@ -206,31 +211,35 @@ public class TieredStoreMetricsManager {
.ofLongs()
.buildWithCallback(measurement -> {
for (FlatMessageFile flatFile : flatFileStore.deepCopyFlatFileToList()) {
try {
MessageQueue mq = flatFile.getMessageQueue();
MessageQueue mq = flatFile.getMessageQueue();
long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
continue;
}
long maxOffset = next.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId());
long maxTimestamp = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), maxOffset - 1);
if (maxTimestamp > 0 && System.currentTimeMillis() - maxTimestamp > TimeUnit.HOURS.toMillis(flatFile.getFileReservedHours())) {
continue;
}
Attributes commitLogAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
.build();
Attributes commitLogAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.COMMIT_LOG.name().toLowerCase())
.build();
Attributes consumeQueueAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
.build();
long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset();
long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset);
if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) {
measurement.record(0, consumeQueueAttributes);
} else {
measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes);
Attributes consumeQueueAttributes = newAttributesBuilder()
.put(LABEL_TOPIC, mq.getTopic())
.put(LABEL_QUEUE_ID, mq.getQueueId())
.put(LABEL_FILE_TYPE, FileSegmentType.CONSUME_QUEUE.name().toLowerCase())
.build();
long consumeQueueDispatchOffset = flatFile.getConsumeQueueMaxOffset();
long consumeQueueDispatchLatency = next.getMessageStoreTimeStamp(mq.getTopic(), mq.getQueueId(), consumeQueueDispatchOffset);
if (maxOffset <= consumeQueueDispatchOffset || consumeQueueDispatchLatency < 0) {
measurement.record(0, consumeQueueAttributes);
} else {
measurement.record(System.currentTimeMillis() - consumeQueueDispatchLatency, consumeQueueAttributes);
}
} catch (ConsumeQueueException e) {
// TODO: handle exception
}
}
});
tools/BUILD.bazel
+3 -1
View File
@@ -40,6 +40,7 @@ java_library(
"@maven//:io_github_aliyunmq_rocketmq_slf4j_api",
"@maven//:io_github_aliyunmq_rocketmq_logback_classic",
"@maven//:org_apache_rocketmq_rocketmq_rocksdb",
"@maven//:com_alibaba_fastjson2_fastjson2",
],
)
@@ -56,7 +57,8 @@ java_library(
"//:test_deps",
"@maven//:org_apache_commons_commons_lang3",
"@maven//:io_netty_netty_all",
"@maven//:commons_cli_commons_cli",
"@maven//:commons_cli_commons_cli",
"@maven//:org_junit_jupiter_junit_jupiter_api",
],
resources = glob(["src/test/resources/*.xml"]),
)