diff --git a/.licenserc.yaml b/.licenserc.yaml index a6c5cd0bb6..3d6205e930 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,12 +34,14 @@ header: - 'src/test/**/*.log' - '*/src/test/resources/META-INF/service/*' - '*/src/main/resources/META-INF/service/*' + - '*/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json' - '**/target/**' - '**/*.iml' - 'docs/**' - 'localbin/**' - 'distribution/LICENSE-BIN' - 'distribution/NOTICE-BIN' + - 'distribution/conf/rmq-proxy.json' comment: on-failure \ No newline at end of file diff --git a/acl/pom.xml b/acl/pom.xml index b5be56f7fa..c4eac5aac1 100644 --- a/acl/pom.xml +++ b/acl/pom.xml @@ -19,6 +19,10 @@ rocketmq-acl ${project.version} + + ${project.groupId} + rocketmq-proto + ${project.groupId} rocketmq-remoting @@ -62,6 +66,10 @@ commons-validator commons-validator + + com.google.protobuf + protobuf-java-util + diff --git a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java index 30a7912a42..3da5cb0398 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java @@ -17,9 +17,10 @@ package org.apache.rocketmq.acl; +import com.google.protobuf.GeneratedMessageV3; import java.util.List; import java.util.Map; - +import org.apache.rocketmq.acl.common.AuthenticationHeader; import org.apache.rocketmq.common.AclConfig; import org.apache.rocketmq.common.DataVersion; import org.apache.rocketmq.common.PlainAccessConfig; @@ -36,6 +37,14 @@ public interface AccessValidator { */ AccessResource parse(RemotingCommand request, String remoteAddr); + /** + * Parse to get the AccessResource from gRPC protocol + * @param messageV3 + * @param header + * @return Plain access resource + */ + AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header); + /** * Validate the access resource. * diff --git a/acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java b/acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java new file mode 100644 index 0000000000..a38d3ec478 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl; + +public interface PermissionChecker { + void check(AccessResource checkedAccess, AccessResource ownedAccess); +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java new file mode 100644 index 0000000000..00112d7374 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl.common; + +import com.google.common.base.MoreObjects; + +public class AuthenticationHeader { + private String remoteAddress; + private String tenantId; + private String namespace; + private String authorization; + private String datetime; + private String sessionToken; + private String requestId; + private String language; + private String clientVersion; + private String protocol; + private int requestCode; + + AuthenticationHeader(final String remoteAddress, final String tenantId, final String namespace, + final String authorization, final String datetime, final String sessionToken, final String requestId, + final String language, final String clientVersion, final String protocol, final int requestCode) { + this.remoteAddress = remoteAddress; + this.tenantId = tenantId; + this.namespace = namespace; + this.authorization = authorization; + this.datetime = datetime; + this.sessionToken = sessionToken; + this.requestId = requestId; + this.language = language; + this.clientVersion = clientVersion; + this.protocol = protocol; + this.requestCode = requestCode; + } + + public static class MetadataHeaderBuilder { + private String remoteAddress; + private String tenantId; + private String namespace; + private String authorization; + private String datetime; + private String sessionToken; + private String requestId; + private String language; + private String clientVersion; + private String protocol; + private int requestCode; + + MetadataHeaderBuilder() { + } + + public AuthenticationHeader.MetadataHeaderBuilder remoteAddress(final String remoteAddress) { + this.remoteAddress = remoteAddress; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder tenantId(final String tenantId) { + this.tenantId = tenantId; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder namespace(final String namespace) { + this.namespace = namespace; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder authorization(final String authorization) { + this.authorization = authorization; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder datetime(final String datetime) { + this.datetime = datetime; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder sessionToken(final String sessionToken) { + this.sessionToken = sessionToken; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder requestId(final String requestId) { + this.requestId = requestId; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder language(final String language) { + this.language = language; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder clientVersion(final String clientVersion) { + this.clientVersion = clientVersion; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder protocol(final String protocol) { + this.protocol = protocol; + return this; + } + + public AuthenticationHeader.MetadataHeaderBuilder requestCode(final int requestCode) { + this.requestCode = requestCode; + return this; + } + + public AuthenticationHeader build() { + return new AuthenticationHeader(this.remoteAddress, this.tenantId, this.namespace, this.authorization, + this.datetime, this.sessionToken, this.requestId, this.language, this.clientVersion, this.protocol, + this.requestCode); + } + } + + public static AuthenticationHeader.MetadataHeaderBuilder builder() { + return new AuthenticationHeader.MetadataHeaderBuilder(); + } + + public String getRemoteAddress() { + return this.remoteAddress; + } + + public String getTenantId() { + return this.tenantId; + } + + public String getNamespace() { + return this.namespace; + } + + public String getAuthorization() { + return this.authorization; + } + + public String getDatetime() { + return this.datetime; + } + + public String getSessionToken() { + return this.sessionToken; + } + + public String getRequestId() { + return this.requestId; + } + + public String getLanguage() { + return this.language; + } + + public String getClientVersion() { + return this.clientVersion; + } + + public String getProtocol() { + return this.protocol; + } + + public int getRequestCode() { + return this.requestCode; + } + + public void setRemoteAddress(final String remoteAddress) { + this.remoteAddress = remoteAddress; + } + + public void setTenantId(final String tenantId) { + this.tenantId = tenantId; + } + + public void setNamespace(final String namespace) { + this.namespace = namespace; + } + + public void setAuthorization(final String authorization) { + this.authorization = authorization; + } + + public void setDatetime(final String datetime) { + this.datetime = datetime; + } + + public void setSessionToken(final String sessionToken) { + this.sessionToken = sessionToken; + } + + public void setRequestId(final String requestId) { + this.requestId = requestId; + } + + public void setLanguage(final String language) { + this.language = language; + } + + public void setClientVersion(final String clientVersion) { + this.clientVersion = clientVersion; + } + + public void setProtocol(final String protocol) { + this.protocol = protocol; + } + + public void setRequestCode(int requestCode) { + this.requestCode = requestCode; + } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("remoteAddress", remoteAddress) + .add("tenantId", tenantId) + .add("namespace", namespace) + .add("authorization", authorization) + .add("datetime", datetime) + .add("sessionToken", sessionToken) + .add("requestId", requestId) + .add("language", language) + .add("clientVersion", clientVersion) + .add("protocol", protocol) + .add("requestCode", requestCode) + .toString(); + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java new file mode 100644 index 0000000000..aa40b6fd33 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl.common; + +import com.google.common.base.MoreObjects; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; + +public class AuthorizationHeader { + private static final String HEADER_SEPARATOR = " "; + private static final String CREDENTIALS_SEPARATOR = "/"; + private static final int AUTH_HEADER_KV_LENGTH = 2; + private static final String CREDENTIAL = "Credential"; + private static final String SIGNED_HEADERS = "SignedHeaders"; + private static final String SIGNATURE = "Signature"; + private String method; + private String accessKey; + private String[] signedHeaders; + private String signature; + + /** + * Parse authorization from gRPC header. + * + * @param header gRPC header string. + * @throws Exception exception. + */ + public AuthorizationHeader(String header) throws DecoderException { + String[] result = header.split(HEADER_SEPARATOR, 2); + if (result.length != 2) { + throw new DecoderException("authorization header is incorrect"); + } + this.method = result[0]; + String[] keyValues = result[1].split(","); + for (String keyValue : keyValues) { + String[] kv = keyValue.trim().split("=", 2); + int kvLength = kv.length; + if (kv.length != AUTH_HEADER_KV_LENGTH) { + throw new DecoderException("authorization keyValues length is incorrect, actual length=" + kvLength); + } + String authItem = kv[0]; + if (CREDENTIAL.equals(authItem)) { + String[] credential = kv[1].split(CREDENTIALS_SEPARATOR); + int credentialActualLength = credential.length; + if (credentialActualLength == 0) { + throw new DecoderException("authorization credential length is incorrect, actual length=" + credentialActualLength); + } + this.accessKey = credential[0]; + continue; + } + if (SIGNED_HEADERS.equals(authItem)) { + this.signedHeaders = kv[1].split(";"); + continue; + } + if (SIGNATURE.equals(authItem)) { + this.signature = this.hexToBase64(kv[1]); + } + } + } + + public String hexToBase64(String input) throws DecoderException { + byte[] bytes = Hex.decodeHex(input); + return Base64.encodeBase64String(bytes); + } + + public String getMethod() { + return this.method; + } + + public String getAccessKey() { + return this.accessKey; + } + + public String[] getSignedHeaders() { + return this.signedHeaders; + } + + public String getSignature() { + return this.signature; + } + + public void setMethod(final String method) { + this.method = method; + } + + public void setAccessKey(final String accessKey) { + this.accessKey = accessKey; + } + + public void setSignedHeaders(final String[] signedHeaders) { + this.signedHeaders = signedHeaders; + } + + public void setSignature(final String signature) { + this.signature = signature; + } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("method", method) + .add("accessKey", accessKey) + .add("signedHeaders", signedHeaders) + .add("signature", signature) + .toString(); + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java index a0cceed8c3..794c163000 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java @@ -16,11 +16,46 @@ */ package org.apache.rocketmq.acl.plain; +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.Message; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.Subscription; +import apache.rocketmq.v2.SubscriptionEntry; +import apache.rocketmq.v2.TelemetryCommand; +import com.google.protobuf.GeneratedMessageV3; +import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import org.apache.commons.codec.DecoderException; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AclUtils; +import org.apache.rocketmq.acl.common.AuthenticationHeader; +import org.apache.rocketmq.acl.common.AuthorizationHeader; +import org.apache.rocketmq.acl.common.Permission; +import org.apache.rocketmq.acl.common.SessionCredentials; import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.remoting.common.RemotingHelper; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class PlainAccessResource implements AccessResource { @@ -55,6 +90,201 @@ public class PlainAccessResource implements AccessResource { public PlainAccessResource() { } + public static PlainAccessResource parse(RemotingCommand request, String remoteAddr) { + PlainAccessResource accessResource = new PlainAccessResource(); + if (remoteAddr != null && remoteAddr.contains(":")) { + accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':'))); + } else { + accessResource.setWhiteRemoteAddress(remoteAddr); + } + + accessResource.setRequestCode(request.getCode()); + + if (request.getExtFields() == null) { + // If request's extFields is null,then return accessResource directly(users can use whiteAddress pattern) + // The following logic codes depend on the request's extFields not to be null. + return accessResource; + } + accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY)); + accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE)); + accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN)); + + try { + switch (request.getCode()) { + case RequestCode.SEND_MESSAGE: + final String topic = request.getExtFields().get("topic"); + if (PlainAccessResource.isRetryTopic(topic)) { + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); + } else { + accessResource.addResourceAndPerm(topic, Permission.PUB); + } + break; + case RequestCode.SEND_MESSAGE_V2: + final String topicV2 = request.getExtFields().get("b"); + if (PlainAccessResource.isRetryTopic(topicV2)) { + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("a")), Permission.SUB); + } else { + accessResource.addResourceAndPerm(topicV2, Permission.PUB); + } + break; + case RequestCode.CONSUMER_SEND_MSG_BACK: + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); + break; + case RequestCode.PULL_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); + accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB); + break; + case RequestCode.QUERY_MESSAGE: + accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); + break; + case RequestCode.HEART_BEAT: + HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class); + for (ConsumerData data : heartbeatData.getConsumerDataSet()) { + accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB); + for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) { + accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB); + } + } + break; + case RequestCode.UNREGISTER_CLIENT: + final UnregisterClientRequestHeader unregisterClientRequestHeader = + (UnregisterClientRequestHeader) request + .decodeCommandCustomHeader(UnregisterClientRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB); + break; + case RequestCode.GET_CONSUMER_LIST_BY_GROUP: + final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = + (GetConsumerListByGroupRequestHeader) request + .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB); + break; + case RequestCode.UPDATE_CONSUMER_OFFSET: + final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = + (UpdateConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); + accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB); + accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB); + break; + default: + break; + + } + } catch (Throwable t) { + throw new AclException(t.getMessage(), t); + } + + // Content + SortedMap map = new TreeMap(); + for (Map.Entry entry : request.getExtFields().entrySet()) { + if (!SessionCredentials.SIGNATURE.equals(entry.getKey()) + && !MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) { + map.put(entry.getKey(), entry.getValue()); + } + } + accessResource.setContent(AclUtils.combineRequestContent(request, map)); + return accessResource; + } + + public static PlainAccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) { + PlainAccessResource accessResource = new PlainAccessResource(); + String remoteAddress = header.getRemoteAddress(); + if (remoteAddress != null && remoteAddress.contains(":")) { + accessResource.setWhiteRemoteAddress(RemotingHelper.parseHostFromAddress(remoteAddress)); + } else { + accessResource.setWhiteRemoteAddress(remoteAddress); + } + try { + AuthorizationHeader authorizationHeader = new AuthorizationHeader(header.getAuthorization()); + accessResource.setAccessKey(authorizationHeader.getAccessKey()); + accessResource.setSignature(authorizationHeader.getSignature()); + } catch (DecoderException e) { + throw new AclException(e.getMessage(), e); + } + accessResource.setSecretToken(header.getSessionToken()); + accessResource.setRequestCode(header.getRequestCode()); + accessResource.setContent(header.getDatetime().getBytes(StandardCharsets.UTF_8)); + + try { + String rpcFullName = messageV3.getDescriptorForType().getFullName(); + if (HeartbeatRequest.getDescriptor().getFullName().equals(rpcFullName)) { + HeartbeatRequest request = (HeartbeatRequest) messageV3; + if (request.hasGroup()) { + accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB); + } + } else if (SendMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) { + SendMessageRequest request = (SendMessageRequest) messageV3; + if (request.getMessagesCount() <= 0) { + throw new AclException("SendMessageRequest, messageCount is zero", ResponseCode.MESSAGE_ILLEGAL); + } + Resource topic = request.getMessages(0).getTopic(); + for (Message message : request.getMessagesList()) { + if (!message.getTopic().equals(topic)) { + throw new AclException("SendMessageRequest, messages' topic is not consistent", ResponseCode.MESSAGE_ILLEGAL); + } + } + accessResource.addResourceAndPerm(topic, Permission.PUB); + } else if (ReceiveMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) { + ReceiveMessageRequest request = (ReceiveMessageRequest) messageV3; + accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB); + accessResource.addResourceAndPerm(request.getMessageQueue().getTopic(), Permission.SUB); + } else if (AckMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) { + AckMessageRequest request = (AckMessageRequest) messageV3; + accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB); + accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB); + } else if (ForwardMessageToDeadLetterQueueRequest.getDescriptor().getFullName().equals(rpcFullName)) { + ForwardMessageToDeadLetterQueueRequest request = (ForwardMessageToDeadLetterQueueRequest) messageV3; + accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB); + accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB); + } else if (EndTransactionRequest.getDescriptor().getFullName().equals(rpcFullName)) { + EndTransactionRequest request = (EndTransactionRequest) messageV3; + accessResource.addResourceAndPerm(request.getTopic(), Permission.PUB); + } else if (TelemetryCommand.getDescriptor().getFullName().equals(rpcFullName)) { + TelemetryCommand command = (TelemetryCommand) messageV3; + if (command.getCommandCase() == TelemetryCommand.CommandCase.SETTINGS) { + if (command.getSettings().hasPublishing()) { + List topicList = command.getSettings().getPublishing().getTopicsList(); + for (Resource topic : topicList) { + accessResource.addResourceAndPerm(topic, Permission.PUB); + } + } + if (command.getSettings().hasSubscription()) { + Subscription subscription = command.getSettings().getSubscription(); + accessResource.addResourceAndPerm(subscription.getGroup(), Permission.SUB); + for (SubscriptionEntry entry : subscription.getSubscriptionsList()) { + accessResource.addResourceAndPerm(entry.getTopic(), Permission.SUB); + } + } + } + } + } catch (Throwable t) { + throw new AclException(t.getMessage(), t); + } + return accessResource; + } + + private void addResourceAndPerm(Resource resource, byte permission) { + String resourceName = NamespaceUtil.wrapNamespace(resource.getResourceNamespace(), resource.getName()); + addResourceAndPerm(resourceName, permission); + } + + public static PlainAccessResource build(PlainAccessConfig plainAccessConfig, RemoteAddressStrategy remoteAddressStrategy) { + PlainAccessResource plainAccessResource = new PlainAccessResource(); + plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); + plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey()); + plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress()); + + plainAccessResource.setAdmin(plainAccessConfig.isAdmin()); + + plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm())); + plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm())); + + Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms()); + Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); + + plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategy); + return plainAccessResource; + } + public static boolean isRetryTopic(String topic) { return null != topic && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX); } diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java index 3889d77c10..1c45c2cf09 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java @@ -16,31 +16,16 @@ */ package org.apache.rocketmq.acl.plain; -import org.apache.rocketmq.acl.AccessResource; -import org.apache.rocketmq.acl.AccessValidator; -import org.apache.rocketmq.acl.common.AclException; -import org.apache.rocketmq.acl.common.AclUtils; -import org.apache.rocketmq.acl.common.Permission; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.common.AclConfig; -import org.apache.rocketmq.common.DataVersion; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.PlainAccessConfig; -import org.apache.rocketmq.common.protocol.RequestCode; -import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; -import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader; -import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; -import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData; -import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; -import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.remoting.protocol.RemotingCommand; - +import com.google.protobuf.GeneratedMessageV3; import java.util.List; import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import static org.apache.rocketmq.acl.plain.PlainAccessResource.getRetryTopic; +import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.acl.AccessValidator; +import org.apache.rocketmq.acl.common.AuthenticationHeader; +import org.apache.rocketmq.common.AclConfig; +import org.apache.rocketmq.common.DataVersion; +import org.apache.rocketmq.common.PlainAccessConfig; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class PlainAccessValidator implements AccessValidator { @@ -52,98 +37,12 @@ public class PlainAccessValidator implements AccessValidator { @Override public AccessResource parse(RemotingCommand request, String remoteAddr) { - PlainAccessResource accessResource = new PlainAccessResource(); - if (remoteAddr != null && remoteAddr.contains(":")) { - accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':'))); - } else { - accessResource.setWhiteRemoteAddress(remoteAddr); - } + return PlainAccessResource.parse(request, remoteAddr); + } - accessResource.setRequestCode(request.getCode()); - - if (request.getExtFields() == null) { - // If request's extFields is null,then return accessResource directly(users can use whiteAddress pattern) - // The following logic codes depend on the request's extFields not to be null. - return accessResource; - } - accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY)); - accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE)); - accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN)); - - try { - switch (request.getCode()) { - case RequestCode.SEND_MESSAGE: - final String topic = request.getExtFields().get("topic"); - if (PlainAccessResource.isRetryTopic(topic)) { - accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); - } else { - accessResource.addResourceAndPerm(topic, Permission.PUB); - } - break; - case RequestCode.SEND_MESSAGE_V2: - final String topicV2 = request.getExtFields().get("b"); - if (PlainAccessResource.isRetryTopic(topicV2)) { - accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("a")), Permission.SUB); - } else { - accessResource.addResourceAndPerm(topicV2, Permission.PUB); - } - break; - case RequestCode.CONSUMER_SEND_MSG_BACK: - accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB); - break; - case RequestCode.PULL_MESSAGE: - accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); - accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB); - break; - case RequestCode.QUERY_MESSAGE: - accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB); - break; - case RequestCode.HEART_BEAT: - HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class); - for (ConsumerData data : heartbeatData.getConsumerDataSet()) { - accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB); - for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) { - accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB); - } - } - break; - case RequestCode.UNREGISTER_CLIENT: - final UnregisterClientRequestHeader unregisterClientRequestHeader = - (UnregisterClientRequestHeader) request - .decodeCommandCustomHeader(UnregisterClientRequestHeader.class); - accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB); - break; - case RequestCode.GET_CONSUMER_LIST_BY_GROUP: - final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader = - (GetConsumerListByGroupRequestHeader) request - .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class); - accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB); - break; - case RequestCode.UPDATE_CONSUMER_OFFSET: - final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader = - (UpdateConsumerOffsetRequestHeader) request - .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); - accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB); - accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB); - break; - default: - break; - - } - } catch (Throwable t) { - throw new AclException(t.getMessage(), t); - } - - // Content - SortedMap map = new TreeMap(); - for (Map.Entry entry : request.getExtFields().entrySet()) { - if (!SessionCredentials.SIGNATURE.equals(entry.getKey()) - && !MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) { - map.put(entry.getKey(), entry.getValue()); - } - } - accessResource.setContent(AclUtils.combineRequestContent(request, map)); - return accessResource; + @Override + public AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) { + return PlainAccessResource.parse(messageV3, header); } @Override @@ -161,22 +60,26 @@ public class PlainAccessValidator implements AccessValidator { return aclPlugEngine.deleteAccessConfig(accesskey); } - @Override public String getAclConfigVersion() { + @Override + public String getAclConfigVersion() { return aclPlugEngine.getAclConfigDataVersion(); } - @Override public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList) { + @Override + public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList) { return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList); } - @Override public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList, String aclFileFullPath) { + @Override + public boolean updateGlobalWhiteAddrsConfig(List globalWhiteAddrsList, String aclFileFullPath) { return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, aclFileFullPath); } - @Override public AclConfig getAllAclConfig() { + @Override + public AclConfig getAllAclConfig() { return aclPlugEngine.getAllAclConfig(); } - + @Override public Map getAllAclConfigVersion() { return aclPlugEngine.getDataVersionMap(); diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java new file mode 100644 index 0000000000..549c9fdce9 --- /dev/null +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.acl.plain; + +import java.util.Map; +import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.acl.PermissionChecker; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.Permission; + +public class PlainPermissionChecker implements PermissionChecker { + public void check(AccessResource checkedAccess, AccessResource ownedAccess) { + PlainAccessResource checkedPlainAccess = (PlainAccessResource) checkedAccess; + PlainAccessResource ownedPlainAccess = (PlainAccessResource) ownedAccess; + if (Permission.needAdminPerm(checkedPlainAccess.getRequestCode()) && !ownedPlainAccess.isAdmin()) { + throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", checkedPlainAccess.getRequestCode(), ownedPlainAccess.getAccessKey())); + } + Map needCheckedPermMap = checkedPlainAccess.getResourcePermMap(); + Map ownedPermMap = ownedPlainAccess.getResourcePermMap(); + + if (needCheckedPermMap == null) { + // If the needCheckedPermMap is null,then return + return; + } + + if (ownedPermMap == null && ownedPlainAccess.isAdmin()) { + // If the ownedPermMap is null and it is an admin user, then return + return; + } + + for (Map.Entry needCheckedEntry : needCheckedPermMap.entrySet()) { + String resource = needCheckedEntry.getKey(); + Byte neededPerm = needCheckedEntry.getValue(); + boolean isGroup = PlainAccessResource.isRetryTopic(resource); + + if (ownedPermMap == null || !ownedPermMap.containsKey(resource)) { + // Check the default perm + byte ownedPerm = isGroup ? ownedPlainAccess.getDefaultGroupPerm() : + ownedPlainAccess.getDefaultTopicPerm(); + if (!Permission.checkPermission(neededPerm, ownedPerm)) { + throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); + } + continue; + } + if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) { + throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); + } + } + } +} diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java index fc8fd09594..7dfcd61ec4 100644 --- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java @@ -37,6 +37,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.acl.PermissionChecker; import org.apache.rocketmq.acl.common.AclConstants; import org.apache.rocketmq.acl.common.AclException; import org.apache.rocketmq.acl.common.AclUtils; @@ -81,6 +82,8 @@ public class PlainPermissionManager { private List fileList = new ArrayList<>(); + private final PermissionChecker permissionChecker = new PlainPermissionChecker(); + public PlainPermissionManager() { this.defaultAclDir = MixAll.dealFilePath(fileHome + File.separator + "conf" + File.separator + "acl"); this.defaultAclFile = MixAll.dealFilePath(fileHome + File.separator + System.getProperty("rocketmq.acl.plain.file", "conf/plain_acl.yml")); @@ -576,40 +579,7 @@ public class PlainPermissionManager { } void checkPerm(PlainAccessResource needCheckedAccess, PlainAccessResource ownedAccess) { - if (Permission.needAdminPerm(needCheckedAccess.getRequestCode()) && !ownedAccess.isAdmin()) { - throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", needCheckedAccess.getRequestCode(), ownedAccess.getAccessKey())); - } - Map needCheckedPermMap = needCheckedAccess.getResourcePermMap(); - Map ownedPermMap = ownedAccess.getResourcePermMap(); - - if (needCheckedPermMap == null) { - // If the needCheckedPermMap is null,then return - return; - } - - if (ownedPermMap == null && ownedAccess.isAdmin()) { - // If the ownedPermMap is null and it is an admin user, then return - return; - } - - for (Map.Entry needCheckedEntry : needCheckedPermMap.entrySet()) { - String resource = needCheckedEntry.getKey(); - Byte neededPerm = needCheckedEntry.getValue(); - boolean isGroup = PlainAccessResource.isRetryTopic(resource); - - if (ownedPermMap == null || !ownedPermMap.containsKey(resource)) { - // Check the default perm - byte ownedPerm = isGroup ? ownedAccess.getDefaultGroupPerm() : - ownedAccess.getDefaultTopicPerm(); - if (!Permission.checkPermission(neededPerm, ownedPerm)) { - throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); - } - continue; - } - if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) { - throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup))); - } - } + permissionChecker.check(needCheckedAccess, ownedAccess); } void clearPermissionInfo() { @@ -631,23 +601,8 @@ public class PlainPermissionManager { public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException { checkPlainAccessConfig(plainAccessConfig); - PlainAccessResource plainAccessResource = new PlainAccessResource(); - plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey()); - plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey()); - plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress()); - - plainAccessResource.setAdmin(plainAccessConfig.isAdmin()); - - plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm())); - plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm())); - - Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms()); - Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms()); - - plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory. - getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress())); - - return plainAccessResource; + return PlainAccessResource.build(plainAccessConfig, remoteAddressStrategyFactory. + getRemoteAddressStrategy(plainAccessConfig.getWhiteRemoteAddress())); } public void validate(PlainAccessResource plainAccessResource) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java index 07b007071c..fe7d452b81 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java @@ -185,6 +185,7 @@ public class BrokerController { protected final Broker2Client broker2Client; protected final SubscriptionGroupManager subscriptionGroupManager; protected final ConsumerIdsChangeListener consumerIdsChangeListener; + protected final EndTransactionProcessor endTransactionProcessor; private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager(); protected BrokerOuterAPI brokerOuterAPI; protected ScheduledExecutorService scheduledExecutorService; @@ -317,6 +318,7 @@ public class BrokerController { this.queryAssignmentProcessor = new QueryAssignmentProcessor(this); this.clientManageProcessor = new ClientManageProcessor(this); this.slaveSynchronize = new SlaveSynchronize(this); + this.endTransactionProcessor = new EndTransactionProcessor(this); this.sendThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); this.putThreadPoolQueue = new LinkedBlockingQueue(this.brokerConfig.getPutThreadPoolQueueCapacity()); @@ -1002,8 +1004,8 @@ public class BrokerController { /** * EndTransactionProcessor */ - this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor); - this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor); + this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor); + this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor); /* * Default @@ -1144,6 +1146,14 @@ public class BrokerController { return popMessageProcessor; } + public AckMessageProcessor getAckMessageProcessor() { + return ackMessageProcessor; + } + + public ChangeInvisibleTimeProcessor getChangeInvisibleTimeProcessor() { + return changeInvisibleTimeProcessor; + } + protected void shutdownBasicService() { shutdown = true; @@ -2036,6 +2046,10 @@ public class BrokerController { return assignmentManager; } + public ClientManageProcessor getClientManageProcessor() { + return clientManageProcessor; + } + public SendMessageProcessor getSendMessageProcessor() { return sendMessageProcessor; } @@ -2096,6 +2110,10 @@ public class BrokerController { return brokerPreOnlineService; } + public EndTransactionProcessor getEndTransactionProcessor() { + return endTransactionProcessor; + } + public boolean isScheduleServiceStart() { return isScheduleServiceStart; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java index 717fb7085e..6c0a58db06 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java @@ -29,5 +29,13 @@ public enum ConsumerGroupEvent { /** * The group of consumer is registered. */ - REGISTER + REGISTER, + /** + * The client of this consumer is new registered. + */ + CLIENT_REGISTER, + /** + * The client of this consumer is unregistered. + */ + CLIENT_UNREGISTER } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java index 09e1241518..638c522feb 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java @@ -98,23 +98,24 @@ public class ConsumerGroupInfo { return result; } - public void unregisterChannel(final ClientChannelInfo clientChannelInfo) { + public boolean unregisterChannel(final ClientChannelInfo clientChannelInfo) { ClientChannelInfo old = this.channelInfoTable.remove(clientChannelInfo.getChannel()); if (old != null) { log.info("unregister a consumer[{}] from consumerGroupInfo {}", this.groupName, old.toString()); + return true; } + return false; } - public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) { + public ClientChannelInfo doChannelCloseEvent(final String remoteAddr, final Channel channel) { final ClientChannelInfo info = this.channelInfoTable.remove(channel); if (info != null) { log.warn( "NETTY EVENT: remove not active channel[{}] from ConsumerGroupInfo groupChannelTable, consumer group: {}", info.toString(), groupName); - return true; } - return false; + return info; } /** diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java index b3bee7cdc0..6b41714691 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java @@ -18,12 +18,15 @@ package org.apache.rocketmq.broker.client; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import io.netty.channel.Channel; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.stream.Collectors; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; @@ -40,16 +43,17 @@ public class ConsumerManager { private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120; private final ConcurrentMap consumerTable = new ConcurrentHashMap(1024); - private final ConsumerIdsChangeListener consumerIdsChangeListener; + private final List consumerIdsChangeListenerList = new CopyOnWriteArrayList<>(); protected final BrokerStatsManager brokerStatsManager; public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener) { - this.consumerIdsChangeListener = consumerIdsChangeListener; + this.consumerIdsChangeListenerList.add(consumerIdsChangeListener); this.brokerStatsManager = null; } - public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, final BrokerStatsManager brokerStatsManager) { - this.consumerIdsChangeListener = consumerIdsChangeListener; + public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, + final BrokerStatsManager brokerStatsManager) { + this.consumerIdsChangeListenerList.add(consumerIdsChangeListener); this.brokerStatsManager = brokerStatsManager; } @@ -93,18 +97,19 @@ public class ConsumerManager { while (it.hasNext()) { Entry next = it.next(); ConsumerGroupInfo info = next.getValue(); - removed = info.doChannelCloseEvent(remoteAddr, channel); - if (removed) { + ClientChannelInfo clientChannelInfo = info.doChannelCloseEvent(remoteAddr, channel); + if (clientChannelInfo != null) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, next.getKey(), clientChannelInfo, info.getSubscribeTopics()); if (info.getChannelInfoTable().isEmpty()) { ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey()); if (remove != null) { LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}", next.getKey()); - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, next.getKey()); + callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey()); } } - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); + callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel()); } } return removed; @@ -113,9 +118,18 @@ public class ConsumerManager { public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo, ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, final Set subList, boolean isNotifyConsumerIdsChangedEnable) { + return registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, + isNotifyConsumerIdsChangedEnable, true); + } + + public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, + final Set subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) { long start = System.currentTimeMillis(); ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); if (null == consumerGroupInfo) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo, + subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet())); ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere); ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp); consumerGroupInfo = prev != null ? prev : tmp; @@ -124,18 +138,21 @@ public class ConsumerManager { boolean r1 = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere); - boolean r2 = consumerGroupInfo.updateSubscription(subList); + boolean r2 = false; + if (updateSubscription) { + r2 = consumerGroupInfo.updateSubscription(subList); + } if (r1 || r2) { if (isNotifyConsumerIdsChangedEnable) { - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); + callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } if (null != this.brokerStatsManager) { this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start)); } - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList); + callConsumerIdsChangeListener(ConsumerGroupEvent.REGISTER, group, subList); return r1 || r2; } @@ -144,17 +161,20 @@ public class ConsumerManager { boolean isNotifyConsumerIdsChangedEnable) { ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); if (null != consumerGroupInfo) { - consumerGroupInfo.unregisterChannel(clientChannelInfo); + boolean removed = consumerGroupInfo.unregisterChannel(clientChannelInfo); + if (removed) { + callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics()); + } if (consumerGroupInfo.getChannelInfoTable().isEmpty()) { ConsumerGroupInfo remove = this.consumerTable.remove(group); if (remove != null) { LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}", group); - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, group); + callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group); } } if (isNotifyConsumerIdsChangedEnable) { - this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); + callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel()); } } } @@ -177,6 +197,7 @@ public class ConsumerManager { LOGGER.warn( "SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}", RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group); + callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics()); RemotingUtil.closeChannel(clientChannelInfo.getChannel()); itChannel.remove(); } @@ -204,4 +225,18 @@ public class ConsumerManager { } return groups; } + + public void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener) { + consumerIdsChangeListenerList.add(listener); + } + + protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String group, Object... args) { + for (ConsumerIdsChangeListener listener : consumerIdsChangeListenerList) { + try { + listener.handle(event, group, args); + } catch (Throwable t) { + LOGGER.error("err when call consumerIdsChangeListener", t); + } + } + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java index 8e6e667dd3..1d040245fc 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java @@ -90,6 +90,9 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen Collection subscriptionDataList = (Collection) args[0]; this.brokerController.getConsumerFilterManager().register(group, subscriptionDataList); break; + case CLIENT_REGISTER: + case CLIENT_UNREGISTER: + break; default: throw new RuntimeException("Unknown event " + event); } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerChangeListener.java new file mode 100644 index 0000000000..f8183d33fa --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerChangeListener.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client; + +/** + * producer manager will call this listener when something happen + *

+ * event type: {@link ProducerGroupEvent} + */ +public interface ProducerChangeListener { + + void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo); +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerGroupEvent.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerGroupEvent.java new file mode 100644 index 0000000000..cbf27ce61e --- /dev/null +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerGroupEvent.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client; + +public enum ProducerGroupEvent { + /** + * The group of producer is unregistered. + */ + GROUP_UNREGISTER, + /** + * The client of this producer is unregistered. + */ + CLIENT_UNREGISTER +} diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java index c7a52b176b..2589ca1938 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import org.apache.rocketmq.broker.util.PositiveAtomicCounter; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.protocol.body.ProducerInfo; @@ -44,6 +45,7 @@ public class ProducerManager { private final ConcurrentHashMap clientChannelTable = new ConcurrentHashMap<>(); protected final BrokerStatsManager brokerStatsManager; private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter(); + private final List producerChangeListenerList = new CopyOnWriteArrayList<>(); public ProducerManager() { this.brokerStatsManager = null; @@ -94,8 +96,11 @@ public class ProducerManager { } public void scanNotActiveChannel() { - for (final Map.Entry> entry : this.groupChannelTable - .entrySet()) { + Iterator>> iterator = this.groupChannelTable.entrySet().iterator(); + + while (iterator.hasNext()) { + Map.Entry> entry = iterator.next(); + final String group = entry.getKey(); final ConcurrentHashMap chlMap = entry.getValue(); @@ -112,9 +117,16 @@ public class ProducerManager { log.warn( "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); + callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, info); RemotingUtil.closeChannel(info.getChannel()); } } + + if (chlMap.isEmpty()) { + log.warn("SCAN: remove expired channel from ProducerManager groupChannelTable, all clear, group={}", group); + iterator.remove(); + callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null); + } } } @@ -134,6 +146,14 @@ public class ProducerManager { log.info( "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", clientChannelInfo.toString(), remoteAddr, group); + callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo); + if (clientChannelInfoTable.isEmpty()) { + ConcurrentHashMap oldGroupTable = this.groupChannelTable.remove(group); + if (oldGroupTable != null) { + log.info("unregister a producer group[{}] from groupChannelTable", group); + callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null); + } + } } } @@ -172,10 +192,12 @@ public class ProducerManager { if (old != null) { log.info("unregister a producer[{}] from groupChannelTable {}", group, clientChannelInfo.toString()); + callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo); } if (channelTable.isEmpty()) { this.groupChannelTable.remove(group); + callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null); log.info("unregister a producer group[{}] from groupChannelTable", group); } } @@ -224,4 +246,19 @@ public class ProducerManager { public Channel findChannel(String clientId) { return clientChannelTable.get(clientId); } + + private void callProducerChangeListener(ProducerGroupEvent event, String group, + ClientChannelInfo clientChannelInfo) { + for (ProducerChangeListener listener : producerChangeListenerList) { + try { + listener.handle(event, group, clientChannelInfo); + } catch (Throwable t) { + log.error("err when call producerChangeListener", t); + } + } + } + + public void appendProducerChangeListener(ProducerChangeListener producerChangeListener) { + producerChangeListenerList.add(producerChangeListener); + } } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java index 3d12678184..5c235714a6 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java @@ -18,7 +18,6 @@ package org.apache.rocketmq.broker.processor; import io.netty.channel.ChannelHandlerContext; import java.net.SocketAddress; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -32,15 +31,10 @@ import org.apache.rocketmq.broker.mqtrace.SendMessageContext; import org.apache.rocketmq.broker.mqtrace.SendMessageHook; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MQVersion; -import org.apache.rocketmq.common.UtilAll; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageType; -import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; -import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.TopicConfig; import org.apache.rocketmq.common.TopicFilterType; +import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.DBMsgConstants; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.constant.PermName; @@ -48,21 +42,24 @@ import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageExtBrokerInner; +import org.apache.rocketmq.common.message.MessageType; import org.apache.rocketmq.common.protocol.NamespaceUtil; -import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; -import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.sysflag.TopicSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingCommandException; import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.common.message.MessageExtBrokerInner; import org.apache.rocketmq.store.PutMessageResult; import org.apache.rocketmq.store.stats.BrokerStatsManager; @@ -574,102 +571,8 @@ public abstract class AbstractSendMessageProcessor implements NettyRequestProces } } - protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) - throws RemotingCommandException { - - SendMessageRequestHeaderV2 requestHeaderV2 = null; - SendMessageRequestHeader requestHeader = null; - switch (request.getCode()) { - case RequestCode.SEND_BATCH_MESSAGE: - case RequestCode.SEND_MESSAGE_V2: - requestHeaderV2 = - (SendMessageRequestHeaderV2) request - .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); - case RequestCode.SEND_MESSAGE: - if (null == requestHeaderV2) { - requestHeader = - (SendMessageRequestHeader) request - .decodeCommandCustomHeader(SendMessageRequestHeader.class); - } else { - requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); - } - default: - break; - } - return requestHeader; - } - - static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request) - throws RemotingCommandException { - SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2(); - HashMap fields = request.getExtFields(); - if (fields == null) { - throw new RemotingCommandException("the ext fields is null"); - } - - String s = fields.get("a"); - checkNotNull(s, "the custom field is null"); - r.setA(s); - - s = fields.get("b"); - checkNotNull(s, "the custom field is null"); - r.setB(s); - - s = fields.get("c"); - checkNotNull(s, "the custom field is null"); - r.setC(s); - - s = fields.get("d"); - checkNotNull(s, "the custom field is null"); - r.setD(Integer.parseInt(s)); - - s = fields.get("e"); - checkNotNull(s, "the custom field is null"); - r.setE(Integer.parseInt(s)); - - s = fields.get("f"); - checkNotNull(s, "the custom field is null"); - r.setF(Integer.parseInt(s)); - - s = fields.get("g"); - checkNotNull(s, "the custom field is null"); - r.setG(Long.parseLong(s)); - - s = fields.get("h"); - checkNotNull(s, "the custom field is null"); - r.setH(Integer.parseInt(s)); - - s = fields.get("i"); - if (s != null) { - r.setI(s); - } - - s = fields.get("j"); - if (s != null) { - r.setJ(Integer.parseInt(s)); - } - - s = fields.get("k"); - if (s != null) { - r.setK(Boolean.parseBoolean(s)); - } - - s = fields.get("l"); - if (s != null) { - r.setL(Integer.parseInt(s)); - } - - s = fields.get("m"); - if (s != null) { - r.setM(Boolean.parseBoolean(s)); - } - return r; - } - - private static void checkNotNull(String s, String msg) throws RemotingCommandException { - if (s == null) { - throw new RemotingCommandException(msg); - } + protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException { + return SendMessageRequestHeader.parseRequestHeader(request); } protected int randomQueueId(int writeQueueNums) { diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java index 991a7058d9..5ab1647e45 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java @@ -179,6 +179,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { ck.setTopic(requestHeader.getTopic()); ck.setQueueId((byte) queueId); ck.addDiff(0); + ck.setBrokerName(brokerName); msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); msgInner.setQueueId(reviveQid); diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java index 333b20cecd..8ac4a6a54f 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java @@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.broker.BrokerController; -import org.apache.rocketmq.broker.client.ConsumerGroupInfo; import org.apache.rocketmq.broker.filter.ConsumerFilterData; import org.apache.rocketmq.broker.filter.ConsumerFilterManager; import org.apache.rocketmq.broker.filter.ExpressionMessageFilter; @@ -251,7 +250,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { } if (requestHeader.isTimeoutTooMuch()) { - response.setCode(POLLING_TIMEOUT); + response.setCode(ResponseCode.POLLING_TIMEOUT); response.setRemark(String.format("the broker[%s] poping message is timeout too much", this.brokerController.getBrokerConfig().getBrokerIP1())); return response; @@ -304,14 +303,6 @@ public class PopMessageProcessor implements NettyRequestProcessor { requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST))); return response; } - ConsumerGroupInfo consumerGroupInfo = - this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup()); - if (null == consumerGroupInfo) { - POP_LOGGER.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup()); - response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); - response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); - return response; - } if (!subscriptionGroupConfig.isConsumeEnable()) { response.setCode(ResponseCode.NO_PERMISSION); @@ -463,6 +454,8 @@ public class PopMessageProcessor implements NettyRequestProcessor { response = null; } break; + case ResponseCode.POLLING_TIMEOUT: + return response; default: assert false; } diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java index 143889a19c..6a3c2d2b29 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java @@ -42,8 +42,8 @@ public class TransactionalMessageCheckService extends ServiceThread { @Override public void run() { log.info("Start transaction check service thread!"); - long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval(); while (!this.isStopped()) { + long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval(); this.waitForRunning(checkInterval); } log.info("End transaction check service thread!"); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerScannerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerScannerTest.java new file mode 100644 index 0000000000..45a39996ad --- /dev/null +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerScannerTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.broker.client; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ConsumerManagerScannerTest { + private ConsumerManager consumerManager; + private String group = "FooBar"; + private String clientId = "clientId"; + private ClientChannelInfo clientInfo; + private Map> groupEventListMap = new HashMap<>(); + + @Mock + private Channel channel; + + @Before + public void init() { + clientInfo = new ClientChannelInfo(channel, clientId, LanguageCode.JAVA, 0); + + consumerManager = new ConsumerManager(new ConsumerIdsChangeListener() { + @Override + public void handle(ConsumerGroupEvent event, String group, Object... args) { + groupEventListMap.compute(event, (eventKey, dataListVal) -> { + if (dataListVal == null) { + dataListVal = new ArrayList<>(); + } + dataListVal.add(new ConsumerIdsChangeListenerData(event, group, args)); + return dataListVal; + }); + } + + @Override + public void shutdown() { + + } + }); + } + + private static class ConsumerIdsChangeListenerData { + private ConsumerGroupEvent event; + private String group; + private Object[] args; + + public ConsumerIdsChangeListenerData(ConsumerGroupEvent event, String group, Object[] args) { + this.event = event; + this.group = group; + this.args = args; + } + } + + @Test + public void testClientUnregisterEventInDoChannelCloseEvent() { + assertThat(consumerManager.registerConsumer( + group, + clientInfo, + ConsumeType.CONSUME_PASSIVELY, + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + new HashSet<>(), + false + )).isTrue(); + + consumerManager.doChannelCloseEvent("remoteAddr", channel); + + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1); + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class); + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]; + assertThat(clientChannelInfo).isSameAs(clientInfo); + } + + @Test + public void testClientUnregisterEventInUnregisterConsumer() { + assertThat(consumerManager.registerConsumer( + group, + clientInfo, + ConsumeType.CONSUME_PASSIVELY, + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + new HashSet<>(), + false + )).isTrue(); + + consumerManager.unregisterConsumer(group, clientInfo, false); + + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1); + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class); + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]; + assertThat(clientChannelInfo).isSameAs(clientInfo); + } + + @Test + public void testClientUnregisterEventInScanNotActiveChannel() { + assertThat(consumerManager.registerConsumer( + group, + clientInfo, + ConsumeType.CONSUME_PASSIVELY, + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + new HashSet<>(), + false + )).isTrue(); + clientInfo.setLastUpdateTimestamp(0); + when(channel.close()).thenReturn(mock(ChannelFuture.class)); + + consumerManager.scanNotActiveChannel(); + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1); + assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class); + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]; + assertThat(clientChannelInfo).isSameAs(clientInfo); + } +} \ No newline at end of file diff --git a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java index 6c794ac5d3..fd76312941 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java @@ -21,6 +21,7 @@ import io.netty.channel.ChannelFuture; import java.lang.reflect.Field; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import org.apache.rocketmq.remoting.protocol.LanguageCode; import org.junit.Before; import org.junit.Test; @@ -50,6 +51,20 @@ public class ProducerManagerTest { @Test public void scanNotActiveChannel() throws Exception { producerManager.registerProducer(group, clientInfo); + AtomicReference groupRef = new AtomicReference<>(); + AtomicReference clientChannelInfoRef = new AtomicReference<>(); + producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> { + switch (event) { + case GROUP_UNREGISTER: + groupRef.set(group); + break; + case CLIENT_UNREGISTER: + clientChannelInfoRef.set(clientChannelInfo); + break; + default: + break; + } + }); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); assertThat(producerManager.findChannel("clientId")).isNotNull(); Field field = ProducerManager.class.getDeclaredField("CHANNEL_EXPIRED_TIMEOUT"); @@ -58,17 +73,35 @@ public class ProducerManagerTest { clientInfo.setLastUpdateTimestamp(System.currentTimeMillis() - CHANNEL_EXPIRED_TIMEOUT - 10); when(channel.close()).thenReturn(mock(ChannelFuture.class)); producerManager.scanNotActiveChannel(); - assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.getGroupChannelTable().get(group)).isNull(); + assertThat(groupRef.get()).isEqualTo(group); + assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo); assertThat(producerManager.findChannel("clientId")).isNull(); } @Test public void doChannelCloseEvent() throws Exception { producerManager.registerProducer(group, clientInfo); + AtomicReference groupRef = new AtomicReference<>(); + AtomicReference clientChannelInfoRef = new AtomicReference<>(); + producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> { + switch (event) { + case GROUP_UNREGISTER: + groupRef.set(group); + break; + case CLIENT_UNREGISTER: + clientChannelInfoRef.set(clientChannelInfo); + break; + default: + break; + } + }); assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull(); assertThat(producerManager.findChannel("clientId")).isNotNull(); producerManager.doChannelCloseEvent("127.0.0.1", channel); - assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull(); + assertThat(producerManager.getGroupChannelTable().get(group)).isNull(); + assertThat(groupRef.get()).isEqualTo(group); + assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo); assertThat(producerManager.findChannel("clientId")).isNull(); } @@ -86,6 +119,20 @@ public class ProducerManagerTest { @Test public void unregisterProducer() throws Exception { producerManager.registerProducer(group, clientInfo); + AtomicReference groupRef = new AtomicReference<>(); + AtomicReference clientChannelInfoRef = new AtomicReference<>(); + producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> { + switch (event) { + case GROUP_UNREGISTER: + groupRef.set(group); + break; + case CLIENT_UNREGISTER: + clientChannelInfoRef.set(clientChannelInfo); + break; + default: + break; + } + }); Map channelMap = producerManager.getGroupChannelTable().get(group); assertThat(channelMap).isNotNull(); assertThat(channelMap.get(channel)).isEqualTo(clientInfo); @@ -95,6 +142,8 @@ public class ProducerManagerTest { producerManager.unregisterProducer(group, clientInfo); channelMap = producerManager.getGroupChannelTable().get(group); channel1 = producerManager.findChannel("clientId"); + assertThat(groupRef.get()).isEqualTo(group); + assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo); assertThat(channelMap).isNull(); assertThat(channel1).isNull(); diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java index 67b4fb96cb..85582fcff9 100644 --- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java +++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java @@ -110,16 +110,6 @@ public class PopMessageProcessorTest { assertThat(response.getRemark()).contains("topic[" + topic + "] not exist"); } - @Test - public void testProcessRequest_SubNotExist() throws RemotingCommandException { - brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, false); - final RemotingCommand request = createPopMsgCommand(); - RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request); - assertThat(response).isNotNull(); - assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_EXIST); - assertThat(response.getRemark()).contains("consumer's group info not exist"); - } - @Test public void testProcessRequest_Found() throws RemotingCommandException { GetMessageResult getMessageResult = createGetMessageResult(1); diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java index eeb8826730..02f5efac2e 100644 --- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java @@ -35,6 +35,8 @@ import org.apache.rocketmq.remoting.protocol.RequestType; */ public class ClientConfig { public static final String SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY = "com.rocketmq.sendMessageWithVIPChannel"; + public static final String DECODE_READ_BODY = "com.rocketmq.read.body"; + public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body"; private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses(); private String clientIP = RemotingUtil.getLocalAddress(); private String instanceName = System.getProperty("rocketmq.client.name", "DEFAULT"); @@ -58,6 +60,8 @@ public class ClientConfig { private long pullTimeDelayMillsWhenException = 1000; private boolean unitMode = false; private String unitName; + private boolean decodeReadBody = Boolean.parseBoolean(System.getProperty(DECODE_READ_BODY, "true")); + private boolean decodeDecompressBody = Boolean.parseBoolean(System.getProperty(DECODE_DECOMPRESS_BODY, "true")); private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "false")); private boolean useTLS = TlsSystemConfig.tlsEnable; @@ -172,6 +176,8 @@ public class ClientConfig { this.namespace = cc.namespace; this.language = cc.language; this.mqClientApiTimeout = cc.mqClientApiTimeout; + this.decodeReadBody = cc.decodeReadBody; + this.decodeDecompressBody = cc.decodeDecompressBody; this.enableStreamRequestType = cc.enableStreamRequestType; } @@ -192,6 +198,8 @@ public class ClientConfig { cc.namespace = namespace; cc.language = language; cc.mqClientApiTimeout = mqClientApiTimeout; + cc.decodeReadBody = decodeReadBody; + cc.decodeDecompressBody = decodeDecompressBody; cc.enableStreamRequestType = enableStreamRequestType; return cc; } @@ -293,6 +301,22 @@ public class ClientConfig { this.language = language; } + public boolean isDecodeReadBody() { + return decodeReadBody; + } + + public void setDecodeReadBody(boolean decodeReadBody) { + this.decodeReadBody = decodeReadBody; + } + + public boolean isDecodeDecompressBody() { + return decodeDecompressBody; + } + + public void setDecodeDecompressBody(boolean decodeDecompressBody) { + this.decodeDecompressBody = decodeDecompressBody; + } + public String getNamespace() { if (namespaceInitialized) { return namespace; @@ -347,6 +371,7 @@ public class ClientConfig { + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout + + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody + ", enableStreamRequestType=" + enableStreamRequestType + "]"; } } diff --git a/client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java b/client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java index f4534742d5..9bbcce2178 100644 --- a/client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java +++ b/client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java @@ -37,6 +37,13 @@ public class MQClientException extends Exception { this.errorMessage = errorMessage; } + public MQClientException(int responseCode, String errorMessage, Throwable cause) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage), cause); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + public int getResponseCode() { return responseCode; } diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java index 528f5646e2..b0371784ec 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java @@ -31,6 +31,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.lang3.StringUtils; import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.common.ClientErrorCode; import org.apache.rocketmq.client.consumer.AckCallback; import org.apache.rocketmq.client.consumer.AckResult; import org.apache.rocketmq.client.consumer.AckStatus; @@ -735,7 +736,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } } - private SendResult processSendResponse( + protected SendResult processSendResponse( final String brokerName, final Message msg, final RemotingCommand response, @@ -853,9 +854,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } } else { if (!responseFuture.isSendRequestOK()) { - popCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); + popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); } else if (responseFuture.isTimeout()) { - popCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, + popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, responseFuture.getCause())); } else { popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); @@ -892,9 +893,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } } else { if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); + ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, + ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, responseFuture.getCause())); } else { ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); @@ -938,9 +939,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } } else { if (!responseFuture.isSendRequestOK()) { - ackCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); + ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); } else if (responseFuture.isTimeout()) { - ackCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, + ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, responseFuture.getCause())); } else { ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); @@ -970,9 +971,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { } } else { if (!responseFuture.isSendRequestOK()) { - pullCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); + pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); } else if (responseFuture.isTimeout()) { - pullCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, + pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, responseFuture.getCause())); } else { pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); @@ -1029,7 +1030,11 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { case ResponseCode.SUCCESS: popStatus = PopStatus.FOUND; ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody()); - msgFoundList = MessageDecoder.decodes(byteBuffer); + msgFoundList = MessageDecoder.decodesBatch( + byteBuffer, + clientConfig.isDecodeReadBody(), + clientConfig.isDecodeDecompressBody(), + true); break; case ResponseCode.POLLING_FULL: popStatus = PopStatus.POLLING_FULL; diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java index 187b2573dc..689cbc8ee2 100644 --- a/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java +++ b/client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -78,7 +78,12 @@ public class PullAPIWrapper { this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId()); if (PullStatus.FOUND == pullResult.getPullStatus()) { ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); - List msgList = MessageDecoder.decodes(byteBuffer); + List msgList = MessageDecoder.decodesBatch( + byteBuffer, + this.mQClientFactory.getClientConfig().isDecodeReadBody(), + this.mQClientFactory.getClientConfig().isDecodeDecompressBody(), + true + ); boolean needDecodeInnerMessage = false; for (MessageExt messageExt: msgList) { diff --git a/common/pom.xml b/common/pom.xml index 16fe95fcd3..fc810f26cd 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -52,5 +52,14 @@ com.google.guava guava + + org.slf4j + slf4j-api + 1.7.7 + + + commons-codec + commons-codec + diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java index a77d5c2d21..b8ba059ea3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java @@ -44,4 +44,6 @@ public class LoggerName { public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; public static final String STDOUT_LOGGER_NAME = "STDOUT"; + public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; + public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; } diff --git a/common/src/main/java/org/apache/rocketmq/common/consumer/ReceiptHandle.java b/common/src/main/java/org/apache/rocketmq/common/consumer/ReceiptHandle.java new file mode 100644 index 0000000000..392a3ae339 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/consumer/ReceiptHandle.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.consumer; + +import java.util.Arrays; +import java.util.List; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.message.MessageConst; + +public class ReceiptHandle { + private static final String SEPARATOR = MessageConst.KEY_SEPARATOR; + public static final String NORMAL_TOPIC = "0"; + public static final String RETRY_TOPIC = "1"; + private final long startOffset; + private final long retrieveTime; + private final long invisibleTime; + private final long nextVisibleTime; + private final int reviveQueueId; + private final String topicType; + private final String brokerName; + private final int queueId; + private final long offset; + private final long commitLogOffset; + private final String receiptHandle; + + public String encode() { + return startOffset + SEPARATOR + retrieveTime + SEPARATOR + invisibleTime + SEPARATOR + reviveQueueId + + SEPARATOR + topicType + SEPARATOR + brokerName + SEPARATOR + queueId + SEPARATOR + offset + SEPARATOR + + commitLogOffset; + } + + public boolean isExpired() { + return nextVisibleTime <= System.currentTimeMillis(); + } + + public static ReceiptHandle decode(String receiptHandle) { + List dataList = Arrays.asList(receiptHandle.split(SEPARATOR)); + if (dataList.size() < 8) { + throw new IllegalArgumentException("Parse failed, dataList size " + dataList.size()); + } + long startOffset = Long.parseLong(dataList.get(0)); + long retrieveTime = Long.parseLong(dataList.get(1)); + long invisibleTime = Long.parseLong(dataList.get(2)); + int reviveQueueId = Integer.parseInt(dataList.get(3)); + String topicType = dataList.get(4); + String brokerName = dataList.get(5); + int queueId = Integer.parseInt(dataList.get(6)); + long offset = Long.parseLong(dataList.get(7)); + long commitLogOffset = -1L; + if (dataList.size() >= 9) { + commitLogOffset = Long.parseLong(dataList.get(8)); + } + + return new ReceiptHandleBuilder() + .startOffset(startOffset) + .retrieveTime(retrieveTime) + .invisibleTime(invisibleTime) + .reviveQueueId(reviveQueueId) + .topicType(topicType) + .brokerName(brokerName) + .queueId(queueId) + .offset(offset) + .commitLogOffset(commitLogOffset) + .receiptHandle(receiptHandle).build(); + } + + ReceiptHandle(final long startOffset, final long retrieveTime, final long invisibleTime, final long nextVisibleTime, + final int reviveQueueId, final String topicType, final String brokerName, final int queueId, final long offset, + final long commitLogOffset, final String receiptHandle) { + this.startOffset = startOffset; + this.retrieveTime = retrieveTime; + this.invisibleTime = invisibleTime; + this.nextVisibleTime = nextVisibleTime; + this.reviveQueueId = reviveQueueId; + this.topicType = topicType; + this.brokerName = brokerName; + this.queueId = queueId; + this.offset = offset; + this.commitLogOffset = commitLogOffset; + this.receiptHandle = receiptHandle; + } + + public static class ReceiptHandleBuilder { + private long startOffset; + private long retrieveTime; + private long invisibleTime; + private int reviveQueueId; + private String topicType; + private String brokerName; + private int queueId; + private long offset; + private long commitLogOffset; + private String receiptHandle; + + ReceiptHandleBuilder() { + } + + public ReceiptHandle.ReceiptHandleBuilder startOffset(final long startOffset) { + this.startOffset = startOffset; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder retrieveTime(final long retrieveTime) { + this.retrieveTime = retrieveTime; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder invisibleTime(final long invisibleTime) { + this.invisibleTime = invisibleTime; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder reviveQueueId(final int reviveQueueId) { + this.reviveQueueId = reviveQueueId; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder topicType(final String topicType) { + this.topicType = topicType; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder brokerName(final String brokerName) { + this.brokerName = brokerName; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder queueId(final int queueId) { + this.queueId = queueId; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder offset(final long offset) { + this.offset = offset; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder commitLogOffset(final long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + return this; + } + + public ReceiptHandle.ReceiptHandleBuilder receiptHandle(final String receiptHandle) { + this.receiptHandle = receiptHandle; + return this; + } + + public ReceiptHandle build() { + return new ReceiptHandle(this.startOffset, this.retrieveTime, this.invisibleTime, this.retrieveTime + this.invisibleTime, + this.reviveQueueId, this.topicType, this.brokerName, this.queueId, this.offset, this.commitLogOffset, this.receiptHandle); + } + + @Override + public String toString() { + return "ReceiptHandle.ReceiptHandleBuilder(startOffset=" + this.startOffset + ", retrieveTime=" + this.retrieveTime + ", invisibleTime=" + this.invisibleTime + ", reviveQueueId=" + this.reviveQueueId + ", topic=" + this.topicType + ", brokerName=" + this.brokerName + ", queueId=" + this.queueId + ", offset=" + this.offset + ", commitLogOffset=" + this.commitLogOffset + ", receiptHandle=" + this.receiptHandle + ")"; + } + } + + public static ReceiptHandle.ReceiptHandleBuilder builder() { + return new ReceiptHandle.ReceiptHandleBuilder(); + } + + public long getStartOffset() { + return this.startOffset; + } + + public long getRetrieveTime() { + return this.retrieveTime; + } + + public long getInvisibleTime() { + return this.invisibleTime; + } + + public long getNextVisibleTime() { + return this.nextVisibleTime; + } + + public int getReviveQueueId() { + return this.reviveQueueId; + } + + public String getTopicType() { + return this.topicType; + } + + public String getBrokerName() { + return this.brokerName; + } + + public int getQueueId() { + return this.queueId; + } + + public long getOffset() { + return this.offset; + } + + public long getCommitLogOffset() { + return commitLogOffset; + } + + public String getReceiptHandle() { + return this.receiptHandle; + } + + public boolean isRetryTopic() { + return RETRY_TOPIC.equals(topicType); + } + + public String getRealTopic(String topic, String groupName) { + if (isRetryTopic()) { + return KeyBuilder.buildPopRetryTopic(topic, groupName); + } + return topic; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java index a6b801edab..e3104f1656 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java @@ -39,7 +39,7 @@ public class MessageBatch extends Message implements Iterable { return messages.iterator(); } - public static MessageBatch generateFromList(Collection messages) { + public static MessageBatch generateFromList(Collection messages) { assert messages != null; assert messages.size() > 0; List messageList = new ArrayList(messages.size()); diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java index a823466415..0193fddd7e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java +++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java @@ -65,6 +65,10 @@ public class MessageConst { public static final String PROPERTY_REDIRECT = "REDIRECT"; public static final String PROPERTY_INNER_MULTI_DISPATCH = "INNER_MULTI_DISPATCH"; public static final String PROPERTY_INNER_MULTI_QUEUE_OFFSET = "INNER_MULTI_QUEUE_OFFSET"; + public static final String PROPERTY_TRACE_CONTEXT = "TRACE_CONTEXT"; + public static final String PROPERTY_TIMER_DELAY_SEC = "TIMER_DELAY_SEC"; + public static final String PROPERTY_TIMER_DELIVER_MS = "TIMER_DELIVER_MS"; + public static final String PROPERTY_BORN_HOST = "__BORNHOST"; /** * property which name starts with "__RMQ.TRANSIENT." is called transient one that will not stored in broker disks. @@ -123,5 +127,6 @@ public class MessageConst { STRING_HASH_SET.add(PROPERTY_CLUSTER); STRING_HASH_SET.add(PROPERTY_MESSAGE_TYPE); STRING_HASH_SET.add(PROPERTY_INNER_MULTI_QUEUE_OFFSET); + STRING_HASH_SET.add(PROPERTY_BORN_HOST); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/AckMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/AckMessageRequestHeader.java index 02e388ba4c..a8fea34d94 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/AckMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/AckMessageRequestHeader.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -80,6 +81,12 @@ public class AckMessageRequestHeader implements CommandCustomHeader { @Override public String toString() { - return topic + "," + this.consumerGroup + "," + this.queueId + "," + this.offset + "," + this.extraInfo; + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .add("queueId", queueId) + .add("extraInfo", extraInfo) + .add("offset", offset) + .toString(); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ChangeInvisibleTimeRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ChangeInvisibleTimeRequestHeader.java index a586e490cf..918a2304b7 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ChangeInvisibleTimeRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ChangeInvisibleTimeRequestHeader.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -94,4 +95,14 @@ public class ChangeInvisibleTimeRequestHeader implements CommandCustomHeader { this.queueId = queueId; } + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .add("queueId", queueId) + .add("extraInfo", extraInfo) + .add("offset", offset) + .add("invisibleTime", invisibleTime) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java index 6cba71c7e9..d62802c06a 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -76,4 +77,15 @@ public class CheckTransactionStateRequestHeader implements CommandCustomHeader { public void setOffsetMsgId(String offsetMsgId) { this.offsetMsgId = offsetMsgId; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("tranStateTableOffset", tranStateTableOffset) + .add("commitLogOffset", commitLogOffset) + .add("msgId", msgId) + .add("transactionId", transactionId) + .add("offsetMsgId", offsetMsgId) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java index afc017b2a6..3b478f8a11 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -68,4 +69,14 @@ public class CloneGroupOffsetRequestHeader implements CommandCustomHeader { public void setOffline(boolean offline) { this.offline = offline; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("srcGroup", srcGroup) + .add("destGroup", destGroup) + .add("topic", topic) + .add("offline", offline) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java index 7bad63985d..a7dc28e256 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -97,4 +98,17 @@ public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomH public void setGroupSysFlag(Integer groupSysFlag) { this.groupSysFlag = groupSysFlag; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("clientId", clientId) + .add("msgId", msgId) + .add("brokerName", brokerName) + .add("topic", topic) + .add("topicSysFlag", topicSysFlag) + .add("groupSysFlag", groupSysFlag) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java index bd8fbb44ca..3d65f23921 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -98,7 +99,14 @@ public class ConsumerSendMsgBackRequestHeader implements CommandCustomHeader { @Override public String toString() { - return "ConsumerSendMsgBackRequestHeader [group=" + group + ", originTopic=" + originTopic + ", originMsgId=" + originMsgId - + ", delayLevel=" + delayLevel + ", unitMode=" + unitMode + ", maxReconsumeTimes=" + maxReconsumeTimes + "]"; + return MoreObjects.toStringHelper(this) + .add("offset", offset) + .add("group", group) + .add("delayLevel", delayLevel) + .add("originMsgId", originMsgId) + .add("originTopic", originTopic) + .add("unitMode", unitMode) + .add("maxReconsumeTimes", maxReconsumeTimes) + .toString(); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateAccessConfigRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateAccessConfigRequestHeader.java index 36990fcf64..09a2a0c667 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateAccessConfigRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateAccessConfigRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -110,4 +111,18 @@ public class CreateAccessConfigRequestHeader implements CommandCustomHeader { public void setGroupPerms(String groupPerms) { this.groupPerms = groupPerms; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("accessKey", accessKey) + .add("secretKey", secretKey) + .add("whiteRemoteAddress", whiteRemoteAddress) + .add("admin", admin) + .add("defaultTopicPerm", defaultTopicPerm) + .add("defaultGroupPerm", defaultGroupPerm) + .add("topicPerms", topicPerms) + .add("groupPerms", groupPerms) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java index c3c59d4950..43859410ae 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.TopicFilterType; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; @@ -139,4 +140,20 @@ public class CreateTopicRequestHeader implements CommandCustomHeader { public void setAttributes(String attributes) { this.attributes = attributes; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("topic", topic) + .add("defaultTopic", defaultTopic) + .add("readQueueNums", readQueueNums) + .add("writeQueueNums", writeQueueNums) + .add("perm", perm) + .add("topicFilterType", topicFilterType) + .add("topicSysFlag", topicSysFlag) + .add("order", order) + .add("attributes", attributes) + .add("force", force) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java index 87661c320a..80fdc3d4a6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; @@ -118,14 +119,14 @@ public class EndTransactionRequestHeader implements CommandCustomHeader { @Override public String toString() { - return "EndTransactionRequestHeader{" + - "producerGroup='" + producerGroup + '\'' + - ", tranStateTableOffset=" + tranStateTableOffset + - ", commitLogOffset=" + commitLogOffset + - ", commitOrRollback=" + commitOrRollback + - ", fromTransactionCheck=" + fromTransactionCheck + - ", msgId='" + msgId + '\'' + - ", transactionId='" + transactionId + '\'' + - '}'; + return MoreObjects.toStringHelper(this) + .add("producerGroup", producerGroup) + .add("tranStateTableOffset", tranStateTableOffset) + .add("commitLogOffset", commitLogOffset) + .add("commitOrRollback", commitOrRollback) + .add("fromTransactionCheck", fromTransactionCheck) + .add("msgId", msgId) + .add("transactionId", transactionId) + .toString(); } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java index 6ba069e1ff..69a2fc60d0 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -44,4 +45,12 @@ public class GetConsumeStatsRequestHeader implements CommandCustomHeader { public void setTopic(String topic) { this.topic = topic; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java index 3523a52cae..ecab653150 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -36,4 +37,11 @@ public class GetConsumerListByGroupRequestHeader implements CommandCustomHeader public void setConsumerGroup(String consumerGroup) { this.consumerGroup = consumerGroup; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java index 1bbbd900c5..840716f5a3 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -57,4 +58,13 @@ public class GetConsumerRunningInfoRequestHeader implements CommandCustomHeader public void setJstackEnable(boolean jstackEnable) { this.jstackEnable = jstackEnable; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("clientId", clientId) + .add("jstackEnable", jstackEnable) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java index ca26a869c6..0a983fecf6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -57,4 +58,13 @@ public class GetConsumerStatusRequestHeader implements CommandCustomHeader { public void setClientAddr(String clientAddr) { this.clientAddr = clientAddr; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("topic", topic) + .add("group", group) + .add("clientAddr", clientAddr) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java index f58e050da7..f98e8500dd 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -71,4 +72,13 @@ public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader { public void setCommitted(final boolean committed) { this.committed = committed; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("topic", topic) + .add("queueId", queueId) + .add("committed", committed) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java index 70189b74b5..d54c4aa41c 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -53,4 +54,12 @@ public class GetMinOffsetRequestHeader extends TopicQueueRequestHeader { public void setQueueId(Integer queueId) { this.queueId = queueId; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("topic", topic) + .add("queueId", queueId) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PopMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PopMessageRequestHeader.java index 4d151a23e0..a3a186a917 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PopMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PopMessageRequestHeader.java @@ -16,6 +16,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -152,4 +153,21 @@ public class PopMessageRequestHeader implements CommandCustomHeader { public boolean isOrder() { return this.order != null && this.order.booleanValue(); } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .add("queueId", queueId) + .add("maxMsgNums", maxMsgNums) + .add("invisibleTime", invisibleTime) + .add("pollTime", pollTime) + .add("bornTime", bornTime) + .add("initMode", initMode) + .add("expType", expType) + .add("exp", exp) + .add("order", order) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java index 486efdfb55..317dc5f4e6 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import java.util.HashMap; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; @@ -267,4 +268,22 @@ public class PullMessageRequestHeader extends TopicQueueRequestHeader implements public void setMaxMsgBytes(Integer maxMsgBytes) { this.maxMsgBytes = maxMsgBytes; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .add("queueId", queueId) + .add("queueOffset", queueOffset) + .add("maxMsgBytes", maxMsgBytes) + .add("maxMsgNums", maxMsgNums) + .add("sysFlag", sysFlag) + .add("commitOffset", commitOffset) + .add("suspendTimeoutMillis", suspendTimeoutMillis) + .add("subscription", subscription) + .add("subVersion", subVersion) + .add("expressionType", expressionType) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java index c8291d2665..3753e062b7 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -65,4 +66,12 @@ public class SearchOffsetRequestHeader extends TopicQueueRequestHeader { this.timestamp = timestamp; } + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("topic", topic) + .add("queueId", queueId) + .add("timestamp", timestamp) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java index 808bc2d3d5..4fece199df 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java @@ -20,10 +20,14 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; +import java.util.HashMap; +import org.apache.rocketmq.common.protocol.RequestCode; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; public class SendMessageRequestHeader extends TopicQueueRequestHeader { @CFNotNull @@ -163,4 +167,119 @@ public class SendMessageRequestHeader extends TopicQueueRequestHeader { public void setBatch(boolean batch) { this.batch = batch; } + + public static SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException { + SendMessageRequestHeaderV2 requestHeaderV2 = null; + SendMessageRequestHeader requestHeader = null; + switch (request.getCode()) { + case RequestCode.SEND_BATCH_MESSAGE: + case RequestCode.SEND_MESSAGE_V2: + requestHeaderV2 = + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + case RequestCode.SEND_MESSAGE: + if (null == requestHeaderV2) { + requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + } else { + requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); + } + default: + break; + } + return requestHeader; + } + + public static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request) + throws RemotingCommandException { + SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2(); + HashMap fields = request.getExtFields(); + if (fields == null) { + throw new RemotingCommandException("the ext fields is null"); + } + + String s = fields.get("a"); + checkNotNull(s, "the custom field is null"); + r.setA(s); + + s = fields.get("b"); + checkNotNull(s, "the custom field is null"); + r.setB(s); + + s = fields.get("c"); + checkNotNull(s, "the custom field is null"); + r.setC(s); + + s = fields.get("d"); + checkNotNull(s, "the custom field is null"); + r.setD(Integer.parseInt(s)); + + s = fields.get("e"); + checkNotNull(s, "the custom field is null"); + r.setE(Integer.parseInt(s)); + + s = fields.get("f"); + checkNotNull(s, "the custom field is null"); + r.setF(Integer.parseInt(s)); + + s = fields.get("g"); + checkNotNull(s, "the custom field is null"); + r.setG(Long.parseLong(s)); + + s = fields.get("h"); + checkNotNull(s, "the custom field is null"); + r.setH(Integer.parseInt(s)); + + s = fields.get("i"); + if (s != null) { + r.setI(s); + } + + s = fields.get("j"); + if (s != null) { + r.setJ(Integer.parseInt(s)); + } + + s = fields.get("k"); + if (s != null) { + r.setK(Boolean.parseBoolean(s)); + } + + s = fields.get("l"); + if (s != null) { + r.setL(Integer.parseInt(s)); + } + + s = fields.get("m"); + if (s != null) { + r.setM(Boolean.parseBoolean(s)); + } + return r; + } + + private static void checkNotNull(String s, String msg) throws RemotingCommandException { + if (s == null) { + throw new RemotingCommandException(msg); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("producerGroup", producerGroup) + .add("topic", topic) + .add("defaultTopic", defaultTopic) + .add("defaultTopicQueueNums", defaultTopicQueueNums) + .add("queueId", queueId) + .add("sysFlag", sysFlag) + .add("bornTimestamp", bornTimestamp) + .add("flag", flag) + .add("properties", properties) + .add("reconsumeTimes", reconsumeTimes) + .add("unitMode", unitMode) + .add("batch", batch) + .add("maxReconsumeTimes", maxReconsumeTimes) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java index ff9457e283..f4771252eb 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java @@ -20,6 +20,7 @@ package org.apache.rocketmq.common.protocol.header; import java.util.HashMap; import org.apache.rocketmq.remoting.protocol.FastCodesHeader; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.remoting.CommandCustomHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.annotation.CFNullable; @@ -288,4 +289,23 @@ public class SendMessageRequestHeaderV2 implements CommandCustomHeader, FastCode public void setM(boolean m) { this.m = m; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("a", a) + .add("b", b) + .add("c", c) + .add("d", d) + .add("e", e) + .add("f", f) + .add("g", g) + .add("h", h) + .add("i", i) + .add("j", j) + .add("k", k) + .add("l", l) + .add("m", m) + .toString(); + } } \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java index 11eccd5c1e..77af812184 100644 --- a/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java +++ b/common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java @@ -20,6 +20,7 @@ */ package org.apache.rocketmq.common.protocol.header; +import com.google.common.base.MoreObjects; import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader; import org.apache.rocketmq.remoting.annotation.CFNotNull; import org.apache.rocketmq.remoting.exception.RemotingCommandException; @@ -73,4 +74,14 @@ public class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader { public void setCommitOffset(Long commitOffset) { this.commitOffset = commitOffset; } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("consumerGroup", consumerGroup) + .add("topic", topic) + .add("queueId", queueId) + .add("commitOffset", commitOffset) + .toString(); + } } diff --git a/common/src/main/java/org/apache/rocketmq/common/subscription/CustomizedRetryPolicy.java b/common/src/main/java/org/apache/rocketmq/common/subscription/CustomizedRetryPolicy.java index 1fe1983f32..c15e16c546 100644 --- a/common/src/main/java/org/apache/rocketmq/common/subscription/CustomizedRetryPolicy.java +++ b/common/src/main/java/org/apache/rocketmq/common/subscription/CustomizedRetryPolicy.java @@ -48,6 +48,13 @@ public class CustomizedRetryPolicy implements RetryPolicy { TimeUnit.HOURS.toMillis(2) }; + public CustomizedRetryPolicy() { + } + + public CustomizedRetryPolicy(long[] next) { + this.next = next; + } + public long[] getNext() { return next; } diff --git a/common/src/main/java/org/apache/rocketmq/common/subscription/ExponentialRetryPolicy.java b/common/src/main/java/org/apache/rocketmq/common/subscription/ExponentialRetryPolicy.java index f1c15e2967..6f212b591e 100644 --- a/common/src/main/java/org/apache/rocketmq/common/subscription/ExponentialRetryPolicy.java +++ b/common/src/main/java/org/apache/rocketmq/common/subscription/ExponentialRetryPolicy.java @@ -20,14 +20,20 @@ package org.apache.rocketmq.common.subscription; import com.google.common.base.MoreObjects; import java.util.concurrent.TimeUnit; -/** - * next delay time = min(max, initial * multiplier^reconsumeTimes) - */ public class ExponentialRetryPolicy implements RetryPolicy { private long initial = TimeUnit.SECONDS.toMillis(5); private long max = TimeUnit.HOURS.toMillis(2); private long multiplier = 2; + public ExponentialRetryPolicy() { + } + + public ExponentialRetryPolicy(long initial, long max, long multiplier) { + this.initial = initial; + this.max = max; + this.multiplier = multiplier; + } + public long getInitial() { return initial; } diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java new file mode 100644 index 0000000000..e5bb6a394c --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.thread; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.UtilAll; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class ThreadPoolMonitor { + private static InternalLogger jstackLogger = InternalLoggerFactory.getLogger(ThreadPoolMonitor.class); + private static InternalLogger waterMarkLogger = InternalLoggerFactory.getLogger(ThreadPoolMonitor.class); + + private static final List MONITOR_EXECUTOR = new CopyOnWriteArrayList<>(); + private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build() + ); + + private static volatile long threadPoolStatusPeriodTime = TimeUnit.SECONDS.toMillis(3); + private static volatile boolean enablePrintJstack = true; + private static volatile long jstackPeriodTime = 60000; + private static volatile long jstackTime = System.currentTimeMillis(); + + public static void config(InternalLogger jstackLoggerConfig, InternalLogger waterMarkLoggerConfig, + boolean enablePrintJstack, long jstackPeriodTimeConfig, long threadPoolStatusPeriodTimeConfig) { + jstackLogger = jstackLoggerConfig; + waterMarkLogger = waterMarkLoggerConfig; + threadPoolStatusPeriodTime = threadPoolStatusPeriodTimeConfig; + ThreadPoolMonitor.enablePrintJstack = enablePrintJstack; + jstackPeriodTime = jstackPeriodTimeConfig; + } + + public static ThreadPoolExecutor createAndMonitor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + String name, + int queueCapacity) { + return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity, Collections.emptyList()); + } + + public static ThreadPoolExecutor createAndMonitor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + String name, + int queueCapacity, + ThreadPoolStatusMonitor... threadPoolStatusMonitors) { + return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity, + Lists.newArrayList(threadPoolStatusMonitors)); + } + + public static ThreadPoolExecutor createAndMonitor(int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + String name, + int queueCapacity, + List threadPoolStatusMonitors) { + ThreadPoolExecutor executor = new ThreadPoolExecutor( + corePoolSize, + maximumPoolSize, + keepAliveTime, + unit, + new LinkedBlockingQueue<>(queueCapacity), + new ThreadFactoryBuilder().setNameFormat(name + "-%d").build(), + new ThreadPoolExecutor.DiscardOldestPolicy()); + List printers = Lists.newArrayList(new ThreadPoolQueueSizeMonitor(queueCapacity)); + printers.addAll(threadPoolStatusMonitors); + + MONITOR_EXECUTOR.add(ThreadPoolWrapper.builder() + .name(name) + .threadPoolExecutor(executor) + .statusPrinters(printers) + .build()); + return executor; + } + + public static void logThreadPoolStatus() { + for (ThreadPoolWrapper threadPoolWrapper : MONITOR_EXECUTOR) { + List monitors = threadPoolWrapper.getStatusPrinters(); + for (ThreadPoolStatusMonitor monitor : monitors) { + double value = monitor.value(threadPoolWrapper.getThreadPoolExecutor()); + waterMarkLogger.info("\t{}\t{}\t{}", threadPoolWrapper.getName(), + monitor.describe(), + value); + + if (enablePrintJstack) { + if (monitor.needPrintJstack(threadPoolWrapper.getThreadPoolExecutor(), value) && + System.currentTimeMillis() - jstackTime > jstackPeriodTime) { + jstackTime = System.currentTimeMillis(); + jstackLogger.warn("jstack start\n{}", UtilAll.jstack()); + } + } + } + } + } + + public static void init() { + MONITOR_SCHEDULED.scheduleAtFixedRate(ThreadPoolMonitor::logThreadPoolStatus, 20, + threadPoolStatusPeriodTime, TimeUnit.MILLISECONDS); + } + + public static void shutdown() { + MONITOR_SCHEDULED.shutdown(); + } +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolQueueSizeMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolQueueSizeMonitor.java new file mode 100644 index 0000000000..9e2e2f675c --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolQueueSizeMonitor.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.thread; + +import java.util.concurrent.ThreadPoolExecutor; + +public class ThreadPoolQueueSizeMonitor implements ThreadPoolStatusMonitor { + + private final int maxQueueCapacity; + + public ThreadPoolQueueSizeMonitor(int maxQueueCapacity) { + this.maxQueueCapacity = maxQueueCapacity; + } + + @Override + public String describe() { + return "queueSize"; + } + + @Override + public double value(ThreadPoolExecutor executor) { + return executor.getQueue().size(); + } + + @Override + public boolean needPrintJstack(ThreadPoolExecutor executor, double value) { + return value > maxQueueCapacity * 0.85; + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolStatusMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolStatusMonitor.java new file mode 100644 index 0000000000..548fec52ec --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolStatusMonitor.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.thread; + +import java.util.concurrent.ThreadPoolExecutor; + +public interface ThreadPoolStatusMonitor { + + String describe(); + + double value(ThreadPoolExecutor executor); + + boolean needPrintJstack(ThreadPoolExecutor executor, double value); +} \ No newline at end of file diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolWrapper.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolWrapper.java new file mode 100644 index 0000000000..3e5bbfe574 --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolWrapper.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.thread; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; + +public class ThreadPoolWrapper { + private String name; + private ThreadPoolExecutor threadPoolExecutor; + private List statusPrinters; + + ThreadPoolWrapper(final String name, final ThreadPoolExecutor threadPoolExecutor, + final List statusPrinters) { + this.name = name; + this.threadPoolExecutor = threadPoolExecutor; + this.statusPrinters = statusPrinters; + } + + public static class ThreadPoolWrapperBuilder { + private String name; + private ThreadPoolExecutor threadPoolExecutor; + private List statusPrinters; + + ThreadPoolWrapperBuilder() { + } + + public ThreadPoolWrapper.ThreadPoolWrapperBuilder name(final String name) { + this.name = name; + return this; + } + + public ThreadPoolWrapper.ThreadPoolWrapperBuilder threadPoolExecutor( + final ThreadPoolExecutor threadPoolExecutor) { + this.threadPoolExecutor = threadPoolExecutor; + return this; + } + + public ThreadPoolWrapper.ThreadPoolWrapperBuilder statusPrinters( + final List statusPrinters) { + this.statusPrinters = statusPrinters; + return this; + } + + public ThreadPoolWrapper build() { + return new ThreadPoolWrapper(this.name, this.threadPoolExecutor, this.statusPrinters); + } + + @java.lang.Override + public java.lang.String toString() { + return "ThreadPoolWrapper.ThreadPoolWrapperBuilder(name=" + this.name + ", threadPoolExecutor=" + this.threadPoolExecutor + ", statusPrinters=" + this.statusPrinters + ")"; + } + } + + public static ThreadPoolWrapper.ThreadPoolWrapperBuilder builder() { + return new ThreadPoolWrapper.ThreadPoolWrapperBuilder(); + } + + public String getName() { + return this.name; + } + + public ThreadPoolExecutor getThreadPoolExecutor() { + return this.threadPoolExecutor; + } + + public List getStatusPrinters() { + return this.statusPrinters; + } + + public void setName(final String name) { + this.name = name; + } + + public void setThreadPoolExecutor(final ThreadPoolExecutor threadPoolExecutor) { + this.threadPoolExecutor = threadPoolExecutor; + } + + public void setStatusPrinters(final List statusPrinters) { + this.statusPrinters = statusPrinters; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ThreadPoolWrapper wrapper = (ThreadPoolWrapper) o; + return Objects.equal(name, wrapper.name) && Objects.equal(threadPoolExecutor, wrapper.threadPoolExecutor) && Objects.equal(statusPrinters, wrapper.statusPrinters); + } + + @Override public int hashCode() { + return Objects.hashCode(name, threadPoolExecutor, statusPrinters); + } + + @Override public String toString() { + return MoreObjects.toStringHelper(this) + .add("name", name) + .add("threadPoolExecutor", threadPoolExecutor) + .add("statusPrinters", statusPrinters) + .toString(); + } +} diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java new file mode 100644 index 0000000000..421adaca4d --- /dev/null +++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.common.utils; + +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import org.apache.commons.codec.binary.Hex; + +public class BinaryUtil { + public static byte[] calculateMd5(byte[] binaryData) { + MessageDigest messageDigest = null; + try { + messageDigest = MessageDigest.getInstance("MD5"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("MD5 algorithm not found."); + } + messageDigest.update(binaryData); + return messageDigest.digest(); + } + + public static String generateMd5(String bodyStr) { + byte[] bytes = calculateMd5(bodyStr.getBytes(Charset.forName("UTF-8"))); + return Hex.encodeHexString(bytes, false); + } + + public static String generateMd5(byte[] content) { + byte[] bytes = calculateMd5(content); + return Hex.encodeHexString(bytes, false); + } +} \ No newline at end of file diff --git a/distribution/bin/mqproxy b/distribution/bin/mqproxy new file mode 100644 index 0000000000..9f0cb84ea0 --- /dev/null +++ b/distribution/bin/mqproxy @@ -0,0 +1,45 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +sh ${ROCKETMQ_HOME}/bin/runserver.sh org.apache.rocketmq.proxy.ProxyStartup $@ diff --git a/distribution/bin/mqproxy.cmd b/distribution/bin/mqproxy.cmd new file mode 100644 index 0000000000..d5f58e4de3 --- /dev/null +++ b/distribution/bin/mqproxy.cmd @@ -0,0 +1,23 @@ +@echo off +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +if not exist "%ROCKETMQ_HOME%\bin\runserver.cmd" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1 + +call "%ROCKETMQ_HOME%\bin\runserver.cmd" org.apache.rocketmq.proxy.ProxyStartup %* + +IF %ERRORLEVEL% EQU 0 ( + ECHO "Proxy starts OK" +) \ No newline at end of file diff --git a/distribution/bin/mqshutdown b/distribution/bin/mqshutdown index d91fce9c5b..7ea6048e15 100644 --- a/distribution/bin/mqshutdown +++ b/distribution/bin/mqshutdown @@ -58,6 +58,20 @@ case $1 in echo "Send shutdown request to mqnamesrv(${pid}) OK" ;; + proxy) + + pid=`ps ax | grep -i 'org.apache.rocketmq.proxy.ProxyStartup' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No mqproxy running." + exit -1; + fi + + echo "The mqproxy(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to mqproxy(${pid}) OK" + ;; *) - echo "Useage: mqshutdown broker | namesrv" + echo "Useage: mqshutdown broker | namesrv | proxy" esac diff --git a/distribution/conf/logback_proxy.xml b/distribution/conf/logback_proxy.xml new file mode 100644 index 0000000000..ad862d53c7 --- /dev/null +++ b/distribution/conf/logback_proxy.xml @@ -0,0 +1,420 @@ + + + + + + + ${user.home}/logs/rocketmqlogs/proxy.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/proxy.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/proxy_watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/proxy_watermark.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker_default.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker.%i.log.gz + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/protection.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/protection.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/watermark.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/store.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/store.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/remoting.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/remoting.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/storeerror.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/storeerror.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/transaction.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/transaction.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/lock.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/lock.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/filter.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/filter.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/stats.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/stats.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/commercial.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/commercial.%i.log.gz + 1 + 10 + + + 500MB + + + + + ${user.home}/logs/rocketmqlogs/pop.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/pop.%i.log + + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/distribution/conf/rmq-proxy.json b/distribution/conf/rmq-proxy.json new file mode 100644 index 0000000000..077404aaa4 --- /dev/null +++ b/distribution/conf/rmq-proxy.json @@ -0,0 +1,3 @@ +{ + +} \ No newline at end of file diff --git a/distribution/pom.xml b/distribution/pom.xml index 0dfda399ad..ce27e43dea 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -38,6 +38,10 @@ org.apache.rocketmq rocketmq-broker + + org.apache.rocketmq + rocketmq-proxy + org.apache.rocketmq rocketmq-client diff --git a/docs/en/README.md b/docs/en/README.md index 97aef18a2d..e1e569e5a2 100644 --- a/docs/en/README.md +++ b/docs/en/README.md @@ -33,6 +33,9 @@ - [Cluster Deployment](dledger/deploy_guide.md)：introduce how to deploy Dledger in cluster. +- [Proxy Deployment](proxy/deploy_guide.md) + Introduce how to deploy proxy (both `Local` mode and `Cluster` mode). + ### 5. Operation and maintenance management - [Operation](operation.md)：introduce RocketMQ's deployment modes that including single-master mode, multi-master mode, multi-master multi-slave mode and so on, as well as the usage of operation tool mqadmin. diff --git a/docs/en/images/rocketmq_proxy_cluster_mode.png b/docs/en/images/rocketmq_proxy_cluster_mode.png new file mode 100644 index 0000000000..1b4eb5eb31 Binary files /dev/null and b/docs/en/images/rocketmq_proxy_cluster_mode.png differ diff --git a/docs/en/images/rocketmq_proxy_local_mode.png b/docs/en/images/rocketmq_proxy_local_mode.png new file mode 100644 index 0000000000..12e6354a8e Binary files /dev/null and b/docs/en/images/rocketmq_proxy_local_mode.png differ diff --git a/docs/en/proxy/deploy_guide.md b/docs/en/proxy/deploy_guide.md new file mode 100644 index 0000000000..84e5a3c171 --- /dev/null +++ b/docs/en/proxy/deploy_guide.md @@ -0,0 +1,37 @@ +# RocketMQ Proxy Deployment Guide + +## Overview + +RocketMQ Proxy supports two deployment modes, `Local` mode and `Cluster` mode. + +## Configuration + +The configuration applies to both the `Cluster` mode and `Local` mode, whose default path is +distribution/conf/rmq-proxy.json directory. + +## `Cluster` mode + +* Set configuration field `nameSrvAddr`. +* Set configuration field `proxyMode` to `cluster` (case insensitive). + +Run the command below. + +```shell +nohup sh mqproxy & +``` + +The command will only run `Proxy` itself. It requires `Namesrv` and `Broker` components running. + +## `Local` mode + +* Set configuration field `nameSrvAddr`. +* Set configuration field `proxyMode` to `local` (case insensitive). + +Run the command below. + +```shell +nohup sh mqproxy & +``` + +The command will not only run `Proxy`, but also run `Broker`. It requires `Namesrv` only and there's no need for +extra `Broker`. \ No newline at end of file diff --git a/pom.xml b/pom.xml index f416700e31..abffd0a279 100644 --- a/pom.xml +++ b/pom.xml @@ -113,7 +113,7 @@ 0.3.1-alpha 1.2.17 1.30 - 1.9 + 1.13 2.17.1 1.7 1.5.2-2 @@ -124,10 +124,13 @@ 6.0.53 1.0-beta-4 1.4.2 + 2.0.0 + 1.45.0 + 3.20.1 4.13.2 - 2.6.0 + 3.22.0 3.10.0 4.1.0 0.30 @@ -156,7 +159,6 @@ ${project.basedir}/../test/target/jacoco-it.exec file:**/generated-sources/**,**/test/** - @@ -176,6 +178,7 @@ acl example container + proxy @@ -284,9 +287,11 @@ .gitignore .travis.yml + README.md CONTRIBUTING.md bin/README.md .github/** + src/test/resources/** src/test/resources/certs/* src/test/**/*.log src/test/resources/META-INF/service/* @@ -295,6 +300,7 @@ */*.iml docs/** localbin/** + conf/rmq-proxy.json @@ -476,6 +482,11 @@ + + ${project.groupId} + rocketmq-proto + ${rocketmq-proto.version} + ${project.groupId} rocketmq-client @@ -551,6 +562,11 @@ rocketmq-example ${project.version} + + ${project.groupId} + rocketmq-proxy + ${project.version} + org.slf4j slf4j-api @@ -712,13 +728,43 @@ org.awaitility awaitility ${awaitility.version} - test com.google.truth truth ${truth.version} + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + io.grpc + grpc-testing + ${grpc.version} + test + + + com.google.protobuf + protobuf-java-util + ${protobuf-java-util.version} + @@ -727,16 +773,19 @@ junit junit ${junit.version} + test org.assertj assertj-core ${assertj-core.version} + test org.mockito mockito-core ${mockito-core.version} + test org.awaitility diff --git a/proxy/README.md b/proxy/README.md new file mode 100644 index 0000000000..936bd024b0 --- /dev/null +++ b/proxy/README.md @@ -0,0 +1,60 @@ +rocketmq-proxy +-------- + +## Introduction + +`RocketMQ Proxy` is a stateless component that makes full use of the newly introduced `pop` consumption mechanism to +achieve stateless consumption behavior. `gRPC` protocol is supported by `Proxy` now and all the message types +including `normal`, `fifo`, `transaction` and `delay` are supported via `pop` consumption mode. `Proxy` will translate +incoming traffic into customized `Remoting` protocol to access `Broker` and `Namesrv`. + +`Proxy` also handles SSL, authorization/authentication and logging/tracing/metrics and is in charge of connection +management and traffic governance. + +### Multi-language support. + +`gRPC` combined with `Protocol Buffer` makes it easy to implement clients with both `java` and other programming +languages while the server side doesn't need extra work to support different programming languages. +See [rocketmq-clients](https://github.com/apache/rocketmq-clients) for more information. + +### Multi-protocol support. + +With `Proxy` served as a traffic interface, it's convenient to implement multiple protocols upon proxy. `gRPC` protocol +is implemented first and the customized `Remoting` protocol will be implemented later. HTTP/1.1 will also be taken into +consideration. + +## Architecture + +`RocketMQ Proxy` has two deployment modes: `Cluster` mode and `Local` mode. With both modes, `Pop` mode is natively +supported in `Proxy`. + +### `Cluster` mode + +While in `Cluster` mode, `Proxy` is an independent cluster that communicates with `Broker` with remote procedure call. +In this scenario, `Proxy` acts as a stateless computing component while `Broker` is a stateful component with local +storage. This form of deployment introduces the architecture of separation of computing and storage for RocketMQ. + +Due to the separation of computing and storage, `RocketMQ Proxy` can be scaled out indefinitely in `Cluster` mode to +handle traffic peak while `Broker` can focus on storage engine and high availability. + +![](../docs/en/images/rocketmq_proxy_cluster_mode.png) + +### `Local` mode + +`Proxy` in `Local` mode has more similarity with `RocketMQ` 4.x version, which is easily deployed or upgraded for +current RocketMQ users. With `Local` mode, `Proxy` deployed with `Broker` in the same process with inter-process +communication so the network overhead is reduced compared to `Cluster` mode. + +![](../docs/en/images/rocketmq_proxy_local_mode.png) + +## Deploy guide + +See [Proxy Deployment](../docs/en/proxy/deploy_guide.md) + +## Related + +* [rocketmq-apis](https://github.com/apache/rocketmq-apis): Common communication protocol between server and client. +* [rocketmq-clients](https://github.com/apache/rocketmq-clients): Collection of Polyglot Clients for Apache RocketMQ. +* [RIP-37: New and Unified APIs](https://shimo.im/docs/m5kv92OeRRU8olqX): RocketMQ proposal of new and unified APIs + crossing different languages. +* [RIP-39: Support gRPC protocol](https://shimo.im/docs/gXqmeEPYgdUw5bqo): RocketMQ proposal of gRPC protocol support. \ No newline at end of file diff --git a/proxy/pom.xml b/proxy/pom.xml new file mode 100644 index 0000000000..4511bc7fa7 --- /dev/null +++ b/proxy/pom.xml @@ -0,0 +1,89 @@ + + + + + + rocketmq-all + org.apache.rocketmq + 5.0.0-SNAPSHOT + + + 4.0.0 + jar + rocketmq-proxy + rocketmq-proxy ${project.version} + + + 8 + 8 + + + + + org.apache.rocketmq + rocketmq-proto + + + org.apache.rocketmq + rocketmq-broker + + + org.apache.rocketmq + rocketmq-common + + + org.apache.rocketmq + rocketmq-client + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + io.grpc + grpc-services + + + com.google.protobuf + protobuf-java-util + + + org.apache.commons + commons-lang3 + + + org.slf4j + slf4j-api + + + ch.qos.logback + logback-classic + + + + \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyMode.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyMode.java new file mode 100644 index 0000000000..3cc36425b0 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyMode.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy; + +public enum ProxyMode { + LOCAL("LOCAL"), + CLUSTER("CLUSTER"); + + private final String mode; + + ProxyMode(String mode) { + this.mode = mode; + } + + public static boolean isClusterMode(String mode) { + if (mode == null) { + return false; + } + return CLUSTER.mode.equals(mode.toUpperCase()); + } + + public static boolean isClusterMode(ProxyMode mode) { + if (mode == null) { + return false; + } + return CLUSTER.equals(mode); + } + + public static boolean isLocalMode(String mode) { + if (mode == null) { + return false; + } + return LOCAL.mode.equals(mode.toUpperCase()); + } + + public static boolean isLocalMode(ProxyMode mode) { + if (mode == null) { + return false; + } + return LOCAL.equals(mode); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java new file mode 100644 index 0000000000..383a99a5bc --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import java.util.Date; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.BrokerStartup; +import org.apache.rocketmq.client.log.ClientLogger; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.grpc.GrpcServer; +import org.apache.rocketmq.proxy.grpc.GrpcServerBuilder; +import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication; +import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.slf4j.LoggerFactory; + +public class ProxyStartup { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private static final ProxyStartAndShutdown PROXY_START_AND_SHUTDOWN = new ProxyStartAndShutdown(); + + private static class ProxyStartAndShutdown extends AbstractStartAndShutdown { + @Override + public void appendStartAndShutdown(StartAndShutdown startAndShutdown) { + super.appendStartAndShutdown(startAndShutdown); + } + } + + public static void main(String[] args) { + try { + ConfigurationManager.initEnv(); + initLogger(); + ConfigurationManager.intConfig(); + + // init thread pool monitor for proxy. + initThreadPoolMonitor(); + + ThreadPoolExecutor executor = createServerExecutor(); + + MessagingProcessor messagingProcessor = createMessagingProcessor(); + + // create grpcServer + GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor, ConfigurationManager.getProxyConfig().getGrpcServerPort()) + .addService(createServiceProcessor(messagingProcessor)) + .configInterceptor() + .build(); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); + + // start servers one by one. + PROXY_START_AND_SHUTDOWN.start(); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + log.info("try to shutdown server"); + try { + PROXY_START_AND_SHUTDOWN.shutdown(); + } catch (Exception e) { + log.error("err when shutdown rocketmq-proxy", e); + } + })); + } catch (Exception e) { + System.err.println("find an unexpect err." + e); + e.printStackTrace(); + log.error("find an unexpect err.", e); + System.exit(1); + } + + System.out.printf("%s%n", new Date() + " rocketmq-proxy startup successfully"); + log.info(new Date() + " rocketmq-proxy startup successfully"); + } + + private static MessagingProcessor createMessagingProcessor() { + String proxyModeStr = ConfigurationManager.getProxyConfig().getProxyMode(); + MessagingProcessor messagingProcessor; + + if (ProxyMode.isClusterMode(proxyModeStr)) { + messagingProcessor = DefaultMessagingProcessor.createForClusterMode(); + } else if (ProxyMode.isLocalMode(proxyModeStr)) { + BrokerController brokerController = createBrokerController(); + StartAndShutdown brokerControllerWrapper = new StartAndShutdown() { + @Override + public void start() throws Exception { + brokerController.start(); + } + + @Override + public void shutdown() throws Exception { + brokerController.shutdown(); + } + }; + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(brokerControllerWrapper); + messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController); + } else { + throw new IllegalArgumentException("try to start grpc server with wrong mode, use 'local' or 'cluster'"); + } + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(messagingProcessor); + return messagingProcessor; + } + + private static GrpcMessagingApplication createServiceProcessor(MessagingProcessor messagingProcessor) { + GrpcMessagingApplication application = GrpcMessagingApplication.create(messagingProcessor); + PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(application); + return application; + } + + private static BrokerController createBrokerController() { + String[] brokerStartupArgs = new String[] {"-c", ConfigurationManager.getProxyConfig().getBrokerConfigPath()}; + return BrokerStartup.createBrokerController(brokerStartupArgs); + } + + public static ThreadPoolExecutor createServerExecutor() { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + int threadPoolNums = config.getGrpcThreadPoolNums(); + int threadPoolQueueCapacity = config.getGrpcThreadPoolQueueCapacity(); + ThreadPoolExecutor executor = ThreadPoolMonitor.createAndMonitor( + threadPoolNums, + threadPoolNums, + 1, TimeUnit.MINUTES, + "GrpcRequestExecutorThread", + threadPoolQueueCapacity + ); + PROXY_START_AND_SHUTDOWN.appendShutdown(executor::shutdown); + return executor; + } + + public static void initThreadPoolMonitor() { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + ThreadPoolMonitor.config( + InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME), + InternalLoggerFactory.getLogger(LoggerName.PROXY_WATER_MARK_LOGGER_NAME), + config.isEnablePrintJstack(), config.getPrintJstackInMillis(), + config.getPrintThreadPoolStatusInMillis()); + ThreadPoolMonitor.init(); + } + + public static void initLogger() throws JoranException { + System.setProperty("brokerLogDir", ""); + System.setProperty(ClientLogger.CLIENT_LOG_USESLF4J, "true"); + + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + //https://logback.qos.ch/manual/configuration.html + lc.setPackagingDataEnabled(false); + configurator.doConfigure(ConfigurationManager.getProxyHome() + "/conf/logback_proxy.xml"); + } +} \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractCacheLoader.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractCacheLoader.java new file mode 100644 index 0000000000..581caffdbd --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractCacheLoader.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +import com.google.common.cache.CacheLoader; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; +import java.util.concurrent.ThreadPoolExecutor; +import javax.annotation.Nonnull; + +public abstract class AbstractCacheLoader extends CacheLoader { + private final ThreadPoolExecutor cacheRefreshExecutor; + + public AbstractCacheLoader(ThreadPoolExecutor cacheRefreshExecutor) { + this.cacheRefreshExecutor = cacheRefreshExecutor; + } + + @Override + public ListenableFuture reload(@Nonnull K key, @Nonnull V oldValue) throws Exception { + ListenableFutureTask task = ListenableFutureTask.create(() -> { + try { + return getDirectly(key); + } catch (Exception e) { + onErr(key, e); + return oldValue; + } + }); + cacheRefreshExecutor.execute(task); + return task; + } + + @Override + public V load(@Nonnull K key) throws Exception { + return getDirectly(key); + } + + protected abstract V getDirectly(K key) throws Exception; + + protected abstract void onErr(K key, Exception e); +} \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractStartAndShutdown.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractStartAndShutdown.java new file mode 100644 index 0000000000..c59f18c4cf --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractStartAndShutdown.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +public abstract class AbstractStartAndShutdown implements StartAndShutdown { + + protected List startAndShutdownList = new CopyOnWriteArrayList<>(); + + protected void appendStartAndShutdown(StartAndShutdown startAndShutdown) { + this.startAndShutdownList.add(startAndShutdown); + } + + @Override + public void start() throws Exception { + for (StartAndShutdown startAndShutdown : startAndShutdownList) { + startAndShutdown.start(); + } + } + + @Override + public void shutdown() throws Exception { + int index = startAndShutdownList.size() - 1; + for (; index >= 0; index--) { + startAndShutdownList.get(index).shutdown(); + } + } + + public void appendStart(Start start) { + this.appendStartAndShutdown(new StartAndShutdown() { + @Override + public void shutdown() throws Exception { + + } + + @Override + public void start() throws Exception { + start.start(); + } + }); + } + + public void appendShutdown(Shutdown shutdown) { + this.appendStartAndShutdown(new StartAndShutdown() { + @Override + public void shutdown() throws Exception { + shutdown.shutdown(); + } + + @Override + public void start() throws Exception { + + } + }); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java new file mode 100644 index 0000000000..2fc1dab40e --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +import com.google.common.net.HostAndPort; +import java.util.Objects; + +public class Address { + + public enum AddressScheme { + IPv4, + IPv6, + DOMAIN_NAME, + UNRECOGNIZED + } + + private AddressScheme addressScheme; + private HostAndPort hostAndPort; + + public Address(AddressScheme addressScheme, HostAndPort hostAndPort) { + this.addressScheme = addressScheme; + this.hostAndPort = hostAndPort; + } + + public AddressScheme getAddressScheme() { + return addressScheme; + } + + public void setAddressScheme(AddressScheme addressScheme) { + this.addressScheme = addressScheme; + } + + public HostAndPort getHostAndPort() { + return hostAndPort; + } + + public void setHostAndPort(HostAndPort hostAndPort) { + this.hostAndPort = hostAndPort; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Address address = (Address) o; + return addressScheme == address.addressScheme && Objects.equals(hostAndPort, address.hostAndPort); + } + + @Override + public int hashCode() { + return Objects.hash(addressScheme, hostAndPort); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ContextVariable.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ContextVariable.java new file mode 100644 index 0000000000..dcfc529090 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ContextVariable.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +public class ContextVariable { + public static final String REMOTE_ADDRESS = "remote-address"; + public static final String LOCAL_ADDRESS = "local-address"; + public static final String CLIENT_ID = "client-id"; + public static final String LANGUAGE = "language"; + public static final String CLIENT_VERSION = "client-version"; + public static final String REMAINING_MS = "remaining-ms"; + public static final String ACTION = "action"; +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/MessageReceiptHandle.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/MessageReceiptHandle.java new file mode 100644 index 0000000000..64e7a122ab --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/MessageReceiptHandle.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; + +public class MessageReceiptHandle { + private final String group; + private final String topic; + private final int queueId; + private final String messageId; + private final long queueOffset; + private final String originalReceiptHandle; + private final long timestamp; + private final int reconsumeTimes; + private final long expectInvisibleTime; + + private String receiptHandle; + + public MessageReceiptHandle(String group, String topic, int queueId, String receiptHandle, String messageId, + long queueOffset, int reconsumeTimes, long expectInvisibleTime) { + this.group = group; + this.topic = topic; + this.queueId = queueId; + this.receiptHandle = receiptHandle; + this.originalReceiptHandle = receiptHandle; + this.messageId = messageId; + this.queueOffset = queueOffset; + this.reconsumeTimes = reconsumeTimes; + this.expectInvisibleTime = expectInvisibleTime; + this.timestamp = System.currentTimeMillis(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MessageReceiptHandle handle = (MessageReceiptHandle) o; + return queueId == handle.queueId && queueOffset == handle.queueOffset && timestamp == handle.timestamp + && reconsumeTimes == handle.reconsumeTimes && expectInvisibleTime == handle.expectInvisibleTime + && Objects.equal(group, handle.group) && Objects.equal(topic, handle.topic) + && Objects.equal(messageId, handle.messageId) && Objects.equal(originalReceiptHandle, handle.originalReceiptHandle) + && Objects.equal(receiptHandle, handle.receiptHandle); + } + + @Override + public int hashCode() { + return Objects.hashCode(group, topic, queueId, messageId, queueOffset, originalReceiptHandle, timestamp, + reconsumeTimes, expectInvisibleTime, receiptHandle); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("group", group) + .add("topic", topic) + .add("queueId", queueId) + .add("messageId", messageId) + .add("queueOffset", queueOffset) + .add("originalReceiptHandle", originalReceiptHandle) + .add("timestamp", timestamp) + .add("reconsumeTimes", reconsumeTimes) + .add("expectInvisibleTime", expectInvisibleTime) + .add("receiptHandle", receiptHandle) + .toString(); + } + + public String getGroup() { + return group; + } + + public String getTopic() { + return topic; + } + + public int getQueueId() { + return queueId; + } + + public String getReceiptHandle() { + return receiptHandle; + } + + public String getOriginalReceiptHandle() { + return originalReceiptHandle; + } + + public String getMessageId() { + return messageId; + } + + public long getQueueOffset() { + return queueOffset; + } + + public int getReconsumeTimes() { + return reconsumeTimes; + } + + public long getTimestamp() { + return timestamp; + } + + public long getExpectInvisibleTime() { + return expectInvisibleTime; + } + + public void update(String receiptHandle) { + this.receiptHandle = receiptHandle; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyContext.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyContext.java new file mode 100644 index 0000000000..6a35993fec --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyContext.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +import java.util.HashMap; +import java.util.Map; + +public class ProxyContext { + public static final String INNER_ACTION_PREFIX = "Inner"; + private final Map value = new HashMap<>(); + + public static ProxyContext create() { + return new ProxyContext(); + } + + public static ProxyContext createForInner(String actionName) { + return create().setAction(INNER_ACTION_PREFIX + actionName); + } + + public static ProxyContext createForInner(Class clazz) { + return createForInner(clazz.getSimpleName()); + } + + public Map getValue() { + return this.value; + } + + public ProxyContext withVal(String key, Object val) { + this.value.put(key, val); + return this; + } + + public T getVal(String key) { + return (T) this.value.get(key); + } + + public ProxyContext setLocalAddress(String localAddress) { + this.withVal(ContextVariable.LOCAL_ADDRESS, localAddress); + return this; + } + + public String getLocalAddress() { + return this.getVal(ContextVariable.LOCAL_ADDRESS); + } + + public ProxyContext setRemoteAddress(String remoteAddress) { + this.withVal(ContextVariable.REMOTE_ADDRESS, remoteAddress); + return this; + } + + public String getRemoteAddress() { + return this.getVal(ContextVariable.REMOTE_ADDRESS); + } + + public ProxyContext setClientID(String clientID) { + this.withVal(ContextVariable.CLIENT_ID, clientID); + return this; + } + + public String getClientID() { + return this.getVal(ContextVariable.CLIENT_ID); + } + + public ProxyContext setLanguage(String language) { + this.withVal(ContextVariable.LANGUAGE, language); + return this; + } + + public String getLanguage() { + return this.getVal(ContextVariable.LANGUAGE); + } + + public ProxyContext setClientVersion(String clientVersion) { + this.withVal(ContextVariable.CLIENT_VERSION, clientVersion); + return this; + } + + public String getClientVersion() { + return this.getVal(ContextVariable.CLIENT_VERSION); + } + + public ProxyContext setRemainingMs(Long remainingMs) { + this.withVal(ContextVariable.REMAINING_MS, remainingMs); + return this; + } + + public Long getRemainingMs() { + return this.getVal(ContextVariable.REMAINING_MS); + } + + public ProxyContext setAction(String action) { + this.withVal(ContextVariable.ACTION, action); + return this; + } + + public String getAction() { + return this.getVal(ContextVariable.ACTION); + } + +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyException.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyException.java new file mode 100644 index 0000000000..af528329fd --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +public class ProxyException extends RuntimeException { + + private final ProxyExceptionCode code; + + public ProxyException(ProxyExceptionCode code, String message) { + super(message); + this.code = code; + } + + public ProxyException(ProxyExceptionCode code, String message, Throwable cause) { + super(message, cause); + this.code = code; + } + + public ProxyExceptionCode getCode() { + return code; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyExceptionCode.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyExceptionCode.java new file mode 100644 index 0000000000..4f91388215 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyExceptionCode.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common; + +public enum ProxyExceptionCode { + INVALID_BROKER_NAME, + TRANSACTION_DATA_NOT_FOUND, + FORBIDDEN, + MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, + INVALID_RECEIPT_HANDLE, + INTERNAL_SERVER_ERROR, +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java new file mode 100644 index 0000000000..ce68fb2db9 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; + +public class ReceiptHandleGroup { + private final Map> receiptHandleMap = new ConcurrentHashMap<>(); + + public void put(String msgID, String handle, MessageReceiptHandle value) { + Map handleMap = receiptHandleMap.computeIfAbsent(msgID, msgIDKey -> new ConcurrentHashMap<>()); + handleMap.put(handle, value); + } + + public boolean isEmpty() { + return this.receiptHandleMap.isEmpty(); + } + + public MessageReceiptHandle remove(String msgID, String handle) { + AtomicReference resRef = new AtomicReference<>(); + receiptHandleMap.computeIfPresent(msgID, (msgIDKey, handleMap) -> { + resRef.set(handleMap.remove(handle)); + if (handleMap.isEmpty()) { + return null; + } + return handleMap; + }); + return resRef.get(); + } + + public MessageReceiptHandle removeOne(String msgID) { + AtomicReference resRef = new AtomicReference<>(); + receiptHandleMap.computeIfPresent(msgID, (msgIDKey, handleMap) -> { + if (handleMap.isEmpty()) { + return null; + } + Optional handleKey = handleMap.keySet().stream().findAny(); + resRef.set(handleMap.remove(handleKey.get())); + if (handleMap.isEmpty()) { + return null; + } + return handleMap; + }); + return resRef.get(); + } + + public interface DataScanner { + void onData(String msgID, String handle, MessageReceiptHandle receiptHandle); + } + + public void scan(DataScanner scanner) { + this.receiptHandleMap.forEach((msgID, handleMap) -> { + handleMap.forEach((handleStr, v) -> { + scanner.onData(msgID, handleStr, v); + }); + }); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Shutdown.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Shutdown.java new file mode 100644 index 0000000000..28f4f92f54 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Shutdown.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +public interface Shutdown { + void shutdown() throws Exception; +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/Start.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Start.java new file mode 100644 index 0000000000..3cf74d47d2 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/Start.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +public interface Start { + void start() throws Exception; +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/StartAndShutdown.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/StartAndShutdown.java new file mode 100644 index 0000000000..565e92c25c --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/StartAndShutdown.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common; + +public interface StartAndShutdown extends Start, Shutdown { +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java new file mode 100644 index 0000000000..e85360a5da --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common.utils; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; + +public class ExceptionUtils { + + public static Throwable getRealException(Throwable throwable) { + if (throwable instanceof CompletionException || throwable instanceof ExecutionException) { + if (throwable.getCause() != null) { + throwable = throwable.getCause(); + } + } + return throwable; + } + + public static String getErrorDetailMessage(Throwable t) { + if (t == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + sb.append(t.getMessage()).append(". ").append(t.getClass().getSimpleName()); + + if (t.getStackTrace().length > 0) { + sb.append(". ").append(t.getStackTrace()[0]); + } + return sb.toString(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FilterUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FilterUtils.java new file mode 100644 index 0000000000..23eb1e1536 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FilterUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common.utils; + +import java.util.Set; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; + +public class FilterUtils { + /** + * Whether the message's tag matches consumerGroup's SubscriptionData + * + * @param tagsSet, tagSet in {@link SubscriptionData}, tagSet empty means SubscriptionData.SUB_ALL(*) + * @param tags, message's tags, null means not tag attached to the message. + */ + public static boolean isTagMatched(Set tagsSet, String tags) { + if (tagsSet.isEmpty()) { + return true; + } + + if (tags == null) { + return false; + } + + return tagsSet.contains(tags); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java new file mode 100644 index 0000000000..2e194a8cbe --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common.utils; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; + +public class FutureUtils { + + public static CompletableFuture appendNextFuture(CompletableFuture future, + CompletableFuture nextFuture, ExecutorService executor) { + future.whenCompleteAsync((t, throwable) -> { + if (throwable != null) { + nextFuture.completeExceptionally(throwable); + } else { + nextFuture.complete(t); + } + }, executor); + return nextFuture; + } + + public static CompletableFuture addExecutor(CompletableFuture future, ExecutorService executor) { + return appendNextFuture(future, new CompletableFuture<>(), executor); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ProxyUtils.java b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ProxyUtils.java new file mode 100644 index 0000000000..7e82a49613 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ProxyUtils.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.common.utils; + +public class ProxyUtils { + + public static final int MAX_MSG_NUMS_FOR_POP_REQUEST = 32; + + public static final String BROKER_ADDR = "brokerAddr"; +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigFile.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigFile.java new file mode 100644 index 0000000000..37757f8d63 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigFile.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +public interface ConfigFile { + + void initData(); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java new file mode 100644 index 0000000000..cf0b715936 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import com.alibaba.fastjson.JSON; +import java.io.File; +import java.nio.file.Files; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.rocketmq.common.constant.LoggerName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class Configuration { + private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private final AtomicReference proxyConfigReference = new AtomicReference<>(); + + public void init() throws Exception { + String proxyConfigData = loadJsonConfig(ProxyConfig.CONFIG_FILE_NAME); + if (null == proxyConfigData) { + throw new RuntimeException(String.format("load configuration from file: %s error.", ProxyConfig.CONFIG_FILE_NAME)); + } + + ProxyConfig proxyConfig = JSON.parseObject(proxyConfigData, ProxyConfig.class); + proxyConfig.initData(); + setProxyConfig(proxyConfig); + } + + public static String loadJsonConfig(String configFileName) throws Exception { + String filePath = new File(ConfigurationManager.getProxyHome() + File.separator + "conf", configFileName).toString(); + + File file = new File(filePath); + if (!file.exists()) { + log.warn("the config file {} not exist", filePath); + return null; + } + long fileLength = file.length(); + if (fileLength <= 0) { + log.warn("the config file {} length is zero", filePath); + return null; + } + + return new String(Files.readAllBytes(file.toPath())); + } + + public ProxyConfig getProxyConfig() { + return proxyConfigReference.get(); + } + + public void setProxyConfig(ProxyConfig proxyConfig) { + proxyConfigReference.set(proxyConfig); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java new file mode 100644 index 0000000000..61e4498962 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.MixAll; + +public class ConfigurationManager { + public static final String RMQ_PROXY_HOME = "RMQ_PROXY_HOME"; + protected static final String DEFAULT_RMQ_PROXY_HOME = System.getenv(MixAll.ROCKETMQ_HOME_ENV); + protected static String proxyHome; + protected static Configuration configuration; + + public static void initEnv() { + proxyHome = System.getenv(RMQ_PROXY_HOME); + if (StringUtils.isEmpty(proxyHome)) { + proxyHome = System.getProperty(RMQ_PROXY_HOME, DEFAULT_RMQ_PROXY_HOME); + } + } + + public static void intConfig() throws Exception { + configuration = new Configuration(); + configuration.init(); + } + + public static String getProxyHome() { + return proxyHome; + } + + public static ProxyConfig getProxyConfig() { + return configuration.getProxyConfig(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/MetricCollectorMode.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/MetricCollectorMode.java new file mode 100644 index 0000000000..305ca8d28a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/MetricCollectorMode.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.config; + +public enum MetricCollectorMode { + /** + * Do not collect the metric from clients. + */ + OFF(0), + /** + * Collect the metric from clients to the given address. + */ + ON(1), + /** + * Collect the metric by the proxy itself. + */ + PROXY(2); + private final int ordinal; + + MetricCollectorMode(int ordinal) { + this.ordinal = ordinal; + } + + public int getOrdinal() { + return ordinal; + } + + public static MetricCollectorMode getEnumByOrdinal(int ordinal) { + for (MetricCollectorMode mode : MetricCollectorMode.values()) { + if (mode.ordinal == ordinal) { + return mode; + } + } + return OFF; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java new file mode 100644 index 0000000000..bc7c58b6a0 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java @@ -0,0 +1,858 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import java.time.Duration; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.proxy.ProxyMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ProxyConfig implements ConfigFile { + private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + public final static String CONFIG_FILE_NAME = "rmq-proxy.json"; + private static final int PROCESSOR_NUMBER = Runtime.getRuntime().availableProcessors(); + + private String rocketMQClusterName = ""; + + /** + * configuration for ThreadPoolMonitor + */ + private boolean enablePrintJstack = true; + private long printJstackInMillis = Duration.ofSeconds(60).toMillis(); + private long printThreadPoolStatusInMillis = Duration.ofSeconds(3).toMillis(); + + private String nameSrvAddr = ""; + private String nameSrvDomain = ""; + private String nameSrvDomainSubgroup = ""; + /** + * gRPC + */ + private String proxyMode = ProxyMode.CLUSTER.name(); + private Integer grpcServerPort = 8081; + private boolean grpcTlsTestModeEnable = true; + private String grpcTlsKeyPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.key"; + private String grpcTlsCertPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.crt"; + private int grpcBossLoopNum = 1; + private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; + private boolean enableGrpcEpoll = false; + private int grpcThreadPoolNums = 16 + PROCESSOR_NUMBER * 2; + private int grpcThreadPoolQueueCapacity = 100000; + private String brokerConfigPath = ConfigurationManager.getProxyHome() + "/conf/broker.conf"; + /** + * gRPC max message size + * 130M = 4M * 32 messages + 2M attributes + */ + private int grpcMaxInboundMessageSize = 130 * 1024 * 1024; + /** + * max message body size, 0 or negative number means no limit for proxy + */ + private int maxMessageSize = 4 * 1024 * 1024; + /** + * max user property size, 0 or negative number means no limit for proxy + */ + private int maxUserPropertySize = 16 * 1024; + private int userPropertyMaxNum = 128; + /** + * max message group size, 0 or negative number means no limit for proxy + */ + private int maxMessageGroupSize = 64; + private long minInvisibleTimeMillsForRecv = Duration.ofSeconds(10).toMillis(); + private long maxInvisibleTimeMills = Duration.ofHours(12).toMillis(); + private long maxDelayTimeMills = Duration.ofDays(1).toMillis(); + private long maxTransactionRecoverySecond = Duration.ofHours(1).getSeconds(); + private boolean enableTopicMessageTypeCheck = true; + + private int grpcClientProducerMaxAttempts = 3; + private long grpcClientProducerBackoffInitialMillis = 10; + private long grpcClientProducerBackoffMaxMillis = 1000; + private int grpcClientProducerBackoffMultiplier = 2; + private long grpcClientConsumerLongPollingTimeoutMillis = Duration.ofSeconds(30).toMillis(); + private int grpcClientConsumerLongPollingBatchSize = 32; + + private int channelExpiredInSeconds = 60; + private int contextExpiredInSeconds = 30; + + private int rocketmqMQClientNum = 6; + + private long grpcProxyRelayRequestTimeoutInSeconds = 5; + private int grpcProducerThreadPoolNums = PROCESSOR_NUMBER; + private int grpcProducerThreadQueueCapacity = 10000; + private int grpcConsumerThreadPoolNums = PROCESSOR_NUMBER; + private int grpcConsumerThreadQueueCapacity = 10000; + private int grpcRouteThreadPoolNums = PROCESSOR_NUMBER; + private int grpcRouteThreadQueueCapacity = 10000; + private int grpcClientManagerThreadPoolNums = PROCESSOR_NUMBER; + private int grpcClientManagerThreadQueueCapacity = 10000; + private int grpcTransactionThreadPoolNums = PROCESSOR_NUMBER; + private int grpcTransactionThreadQueueCapacity = 10000; + + private int producerProcessorThreadPoolNums = PROCESSOR_NUMBER; + private int producerProcessorThreadPoolQueueCapacity = 10000; + private int consumerProcessorThreadPoolNums = PROCESSOR_NUMBER; + private int consumerProcessorThreadPoolQueueCapacity = 10000; + + private int topicRouteServiceCacheExpiredInSeconds = 20; + private int topicRouteServiceCacheMaxNum = 20000; + private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER; + private int topicRouteServiceThreadPoolQueueCapacity = 5000; + + private int topicConfigCacheExpiredInSeconds = 20; + private int topicConfigCacheMaxNum = 20000; + private int subscriptionGroupConfigCacheExpiredInSeconds = 20; + private int subscriptionGroupConfigCacheMaxNum = 20000; + private int metadataThreadPoolNums = 3; + private int metadataThreadPoolQueueCapacity = 1000; + + private int transactionHeartbeatThreadPoolNums = 20; + private int transactionHeartbeatThreadPoolQueueCapacity = 200; + private int transactionHeartbeatPeriodSecond = 20; + private int transactionHeartbeatBatchNum = 100; + private long transactionDataExpireScanPeriodMillis = Duration.ofSeconds(10).toMillis(); + private long transactionDataMaxWaitClearMillis = Duration.ofSeconds(30).toMillis(); + private long transactionDataExpireMillis = Duration.ofSeconds(30).toMillis(); + private int transactionDataMaxNum = 15; + + private long longPollingReserveTimeInMillis = 100; + + private long invisibleTimeMillisWhenClear = 1000L; + private boolean enableProxyAutoRenew = true; + private long renewAheadTimeMillis = TimeUnit.SECONDS.toMillis(10); + private long renewSliceTimeMillis = TimeUnit.SECONDS.toMillis(60); + private long renewMaxTimeMillis = TimeUnit.HOURS.toMillis(3); + private long renewSchedulePeriodMillis = TimeUnit.SECONDS.toMillis(5); + + private boolean enableACL = false; + + private boolean useDelayLevel = true; + private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h"; + private transient Map delayLevelTable = new ConcurrentHashMap<>(); + + private int metricCollectorMode = MetricCollectorMode.OFF.getOrdinal(); + // Example address: 127.0.0.1:1234 + private String metricCollectorAddress = ""; + + @Override + public void initData() { + parseDelayLevel(); + } + + public int computeDelayLevel(long timeMillis) { + long intervalMillis = timeMillis - System.currentTimeMillis(); + List> sortedLevels = delayLevelTable.entrySet().stream().sorted(Comparator.comparingLong(Map.Entry::getValue)).collect(Collectors.toList()); + for (Map.Entry entry : sortedLevels) { + if (entry.getValue() > intervalMillis) { + return entry.getKey(); + } + } + return sortedLevels.get(sortedLevels.size() - 1).getKey(); + } + + public void parseDelayLevel() { + this.delayLevelTable = new ConcurrentHashMap<>(); + Map timeUnitTable = new HashMap<>(); + timeUnitTable.put("s", 1000L); + timeUnitTable.put("m", 1000L * 60); + timeUnitTable.put("h", 1000L * 60 * 60); + timeUnitTable.put("d", 1000L * 60 * 60 * 24); + + String levelString = this.getMessageDelayLevel(); + try { + String[] levelArray = levelString.split(" "); + for (int i = 0; i < levelArray.length; i++) { + String value = levelArray[i]; + String ch = value.substring(value.length() - 1); + Long tu = timeUnitTable.get(ch); + + int level = i + 1; + long num = Long.parseLong(value.substring(0, value.length() - 1)); + long delayTimeMillis = tu * num; + this.delayLevelTable.put(level, delayTimeMillis); + } + } catch (Exception e) { + log.error("parse delay level failed. messageDelayLevel:{}", messageDelayLevel, e); + } + } + + public String getRocketMQClusterName() { + return rocketMQClusterName; + } + + public void setRocketMQClusterName(String rocketMQClusterName) { + this.rocketMQClusterName = rocketMQClusterName; + } + + public boolean isEnablePrintJstack() { + return enablePrintJstack; + } + + public void setEnablePrintJstack(boolean enablePrintJstack) { + this.enablePrintJstack = enablePrintJstack; + } + + public long getPrintJstackInMillis() { + return printJstackInMillis; + } + + public void setPrintJstackInMillis(long printJstackInMillis) { + this.printJstackInMillis = printJstackInMillis; + } + + public long getPrintThreadPoolStatusInMillis() { + return printThreadPoolStatusInMillis; + } + + public void setPrintThreadPoolStatusInMillis(long printThreadPoolStatusInMillis) { + this.printThreadPoolStatusInMillis = printThreadPoolStatusInMillis; + } + + public String getNameSrvAddr() { + return nameSrvAddr; + } + + public void setNameSrvAddr(String nameSrvAddr) { + this.nameSrvAddr = nameSrvAddr; + } + + public String getNameSrvDomain() { + return nameSrvDomain; + } + + public void setNameSrvDomain(String nameSrvDomain) { + this.nameSrvDomain = nameSrvDomain; + } + + public String getNameSrvDomainSubgroup() { + return nameSrvDomainSubgroup; + } + + public void setNameSrvDomainSubgroup(String nameSrvDomainSubgroup) { + this.nameSrvDomainSubgroup = nameSrvDomainSubgroup; + } + + public String getProxyMode() { + return proxyMode; + } + + public void setProxyMode(String proxyMode) { + this.proxyMode = proxyMode; + } + + public Integer getGrpcServerPort() { + return grpcServerPort; + } + + public void setGrpcServerPort(Integer grpcServerPort) { + this.grpcServerPort = grpcServerPort; + } + + public boolean isGrpcTlsTestModeEnable() { + return grpcTlsTestModeEnable; + } + + public void setGrpcTlsTestModeEnable(boolean grpcTlsTestModeEnable) { + this.grpcTlsTestModeEnable = grpcTlsTestModeEnable; + } + + public String getGrpcTlsKeyPath() { + return grpcTlsKeyPath; + } + + public void setGrpcTlsKeyPath(String grpcTlsKeyPath) { + this.grpcTlsKeyPath = grpcTlsKeyPath; + } + + public String getGrpcTlsCertPath() { + return grpcTlsCertPath; + } + + public void setGrpcTlsCertPath(String grpcTlsCertPath) { + this.grpcTlsCertPath = grpcTlsCertPath; + } + + public int getGrpcBossLoopNum() { + return grpcBossLoopNum; + } + + public void setGrpcBossLoopNum(int grpcBossLoopNum) { + this.grpcBossLoopNum = grpcBossLoopNum; + } + + public int getGrpcWorkerLoopNum() { + return grpcWorkerLoopNum; + } + + public void setGrpcWorkerLoopNum(int grpcWorkerLoopNum) { + this.grpcWorkerLoopNum = grpcWorkerLoopNum; + } + + public boolean isEnableGrpcEpoll() { + return enableGrpcEpoll; + } + + public void setEnableGrpcEpoll(boolean enableGrpcEpoll) { + this.enableGrpcEpoll = enableGrpcEpoll; + } + + public int getGrpcThreadPoolNums() { + return grpcThreadPoolNums; + } + + public void setGrpcThreadPoolNums(int grpcThreadPoolNums) { + this.grpcThreadPoolNums = grpcThreadPoolNums; + } + + public int getGrpcThreadPoolQueueCapacity() { + return grpcThreadPoolQueueCapacity; + } + + public void setGrpcThreadPoolQueueCapacity(int grpcThreadPoolQueueCapacity) { + this.grpcThreadPoolQueueCapacity = grpcThreadPoolQueueCapacity; + } + + public String getBrokerConfigPath() { + return brokerConfigPath; + } + + public void setBrokerConfigPath(String brokerConfigPath) { + this.brokerConfigPath = brokerConfigPath; + } + + public int getGrpcMaxInboundMessageSize() { + return grpcMaxInboundMessageSize; + } + + public void setGrpcMaxInboundMessageSize(int grpcMaxInboundMessageSize) { + this.grpcMaxInboundMessageSize = grpcMaxInboundMessageSize; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + public int getMaxUserPropertySize() { + return maxUserPropertySize; + } + + public void setMaxUserPropertySize(int maxUserPropertySize) { + this.maxUserPropertySize = maxUserPropertySize; + } + + public int getUserPropertyMaxNum() { + return userPropertyMaxNum; + } + + public void setUserPropertyMaxNum(int userPropertyMaxNum) { + this.userPropertyMaxNum = userPropertyMaxNum; + } + + public int getMaxMessageGroupSize() { + return maxMessageGroupSize; + } + + public void setMaxMessageGroupSize(int maxMessageGroupSize) { + this.maxMessageGroupSize = maxMessageGroupSize; + } + + public long getMinInvisibleTimeMillsForRecv() { + return minInvisibleTimeMillsForRecv; + } + + public void setMinInvisibleTimeMillsForRecv(long minInvisibleTimeMillsForRecv) { + this.minInvisibleTimeMillsForRecv = minInvisibleTimeMillsForRecv; + } + + public long getMaxInvisibleTimeMills() { + return maxInvisibleTimeMills; + } + + public void setMaxInvisibleTimeMills(long maxInvisibleTimeMills) { + this.maxInvisibleTimeMills = maxInvisibleTimeMills; + } + + public long getMaxDelayTimeMills() { + return maxDelayTimeMills; + } + + public void setMaxDelayTimeMills(long maxDelayTimeMills) { + this.maxDelayTimeMills = maxDelayTimeMills; + } + + public long getMaxTransactionRecoverySecond() { + return maxTransactionRecoverySecond; + } + + public void setMaxTransactionRecoverySecond(long maxTransactionRecoverySecond) { + this.maxTransactionRecoverySecond = maxTransactionRecoverySecond; + } + + public int getGrpcClientProducerMaxAttempts() { + return grpcClientProducerMaxAttempts; + } + + public void setGrpcClientProducerMaxAttempts(int grpcClientProducerMaxAttempts) { + this.grpcClientProducerMaxAttempts = grpcClientProducerMaxAttempts; + } + + public long getGrpcClientProducerBackoffInitialMillis() { + return grpcClientProducerBackoffInitialMillis; + } + + public void setGrpcClientProducerBackoffInitialMillis(long grpcClientProducerBackoffInitialMillis) { + this.grpcClientProducerBackoffInitialMillis = grpcClientProducerBackoffInitialMillis; + } + + public long getGrpcClientProducerBackoffMaxMillis() { + return grpcClientProducerBackoffMaxMillis; + } + + public void setGrpcClientProducerBackoffMaxMillis(long grpcClientProducerBackoffMaxMillis) { + this.grpcClientProducerBackoffMaxMillis = grpcClientProducerBackoffMaxMillis; + } + + public int getGrpcClientProducerBackoffMultiplier() { + return grpcClientProducerBackoffMultiplier; + } + + public void setGrpcClientProducerBackoffMultiplier(int grpcClientProducerBackoffMultiplier) { + this.grpcClientProducerBackoffMultiplier = grpcClientProducerBackoffMultiplier; + } + + public long getGrpcClientConsumerLongPollingTimeoutMillis() { + return grpcClientConsumerLongPollingTimeoutMillis; + } + + public void setGrpcClientConsumerLongPollingTimeoutMillis(long grpcClientConsumerLongPollingTimeoutMillis) { + this.grpcClientConsumerLongPollingTimeoutMillis = grpcClientConsumerLongPollingTimeoutMillis; + } + + public int getGrpcClientConsumerLongPollingBatchSize() { + return grpcClientConsumerLongPollingBatchSize; + } + + public void setGrpcClientConsumerLongPollingBatchSize(int grpcClientConsumerLongPollingBatchSize) { + this.grpcClientConsumerLongPollingBatchSize = grpcClientConsumerLongPollingBatchSize; + } + + public int getChannelExpiredInSeconds() { + return channelExpiredInSeconds; + } + + public void setChannelExpiredInSeconds(int channelExpiredInSeconds) { + this.channelExpiredInSeconds = channelExpiredInSeconds; + } + + public int getContextExpiredInSeconds() { + return contextExpiredInSeconds; + } + + public void setContextExpiredInSeconds(int contextExpiredInSeconds) { + this.contextExpiredInSeconds = contextExpiredInSeconds; + } + + public int getRocketmqMQClientNum() { + return rocketmqMQClientNum; + } + + public void setRocketmqMQClientNum(int rocketmqMQClientNum) { + this.rocketmqMQClientNum = rocketmqMQClientNum; + } + + public long getGrpcProxyRelayRequestTimeoutInSeconds() { + return grpcProxyRelayRequestTimeoutInSeconds; + } + + public void setGrpcProxyRelayRequestTimeoutInSeconds(long grpcProxyRelayRequestTimeoutInSeconds) { + this.grpcProxyRelayRequestTimeoutInSeconds = grpcProxyRelayRequestTimeoutInSeconds; + } + + public int getGrpcProducerThreadPoolNums() { + return grpcProducerThreadPoolNums; + } + + public void setGrpcProducerThreadPoolNums(int grpcProducerThreadPoolNums) { + this.grpcProducerThreadPoolNums = grpcProducerThreadPoolNums; + } + + public int getGrpcProducerThreadQueueCapacity() { + return grpcProducerThreadQueueCapacity; + } + + public void setGrpcProducerThreadQueueCapacity(int grpcProducerThreadQueueCapacity) { + this.grpcProducerThreadQueueCapacity = grpcProducerThreadQueueCapacity; + } + + public int getGrpcConsumerThreadPoolNums() { + return grpcConsumerThreadPoolNums; + } + + public void setGrpcConsumerThreadPoolNums(int grpcConsumerThreadPoolNums) { + this.grpcConsumerThreadPoolNums = grpcConsumerThreadPoolNums; + } + + public int getGrpcConsumerThreadQueueCapacity() { + return grpcConsumerThreadQueueCapacity; + } + + public void setGrpcConsumerThreadQueueCapacity(int grpcConsumerThreadQueueCapacity) { + this.grpcConsumerThreadQueueCapacity = grpcConsumerThreadQueueCapacity; + } + + public int getGrpcRouteThreadPoolNums() { + return grpcRouteThreadPoolNums; + } + + public void setGrpcRouteThreadPoolNums(int grpcRouteThreadPoolNums) { + this.grpcRouteThreadPoolNums = grpcRouteThreadPoolNums; + } + + public int getGrpcRouteThreadQueueCapacity() { + return grpcRouteThreadQueueCapacity; + } + + public void setGrpcRouteThreadQueueCapacity(int grpcRouteThreadQueueCapacity) { + this.grpcRouteThreadQueueCapacity = grpcRouteThreadQueueCapacity; + } + + public int getGrpcClientManagerThreadPoolNums() { + return grpcClientManagerThreadPoolNums; + } + + public void setGrpcClientManagerThreadPoolNums(int grpcClientManagerThreadPoolNums) { + this.grpcClientManagerThreadPoolNums = grpcClientManagerThreadPoolNums; + } + + public int getGrpcClientManagerThreadQueueCapacity() { + return grpcClientManagerThreadQueueCapacity; + } + + public void setGrpcClientManagerThreadQueueCapacity(int grpcClientManagerThreadQueueCapacity) { + this.grpcClientManagerThreadQueueCapacity = grpcClientManagerThreadQueueCapacity; + } + + public int getGrpcTransactionThreadPoolNums() { + return grpcTransactionThreadPoolNums; + } + + public void setGrpcTransactionThreadPoolNums(int grpcTransactionThreadPoolNums) { + this.grpcTransactionThreadPoolNums = grpcTransactionThreadPoolNums; + } + + public int getGrpcTransactionThreadQueueCapacity() { + return grpcTransactionThreadQueueCapacity; + } + + public void setGrpcTransactionThreadQueueCapacity(int grpcTransactionThreadQueueCapacity) { + this.grpcTransactionThreadQueueCapacity = grpcTransactionThreadQueueCapacity; + } + + public int getProducerProcessorThreadPoolNums() { + return producerProcessorThreadPoolNums; + } + + public void setProducerProcessorThreadPoolNums(int producerProcessorThreadPoolNums) { + this.producerProcessorThreadPoolNums = producerProcessorThreadPoolNums; + } + + public int getProducerProcessorThreadPoolQueueCapacity() { + return producerProcessorThreadPoolQueueCapacity; + } + + public void setProducerProcessorThreadPoolQueueCapacity(int producerProcessorThreadPoolQueueCapacity) { + this.producerProcessorThreadPoolQueueCapacity = producerProcessorThreadPoolQueueCapacity; + } + + public int getConsumerProcessorThreadPoolNums() { + return consumerProcessorThreadPoolNums; + } + + public void setConsumerProcessorThreadPoolNums(int consumerProcessorThreadPoolNums) { + this.consumerProcessorThreadPoolNums = consumerProcessorThreadPoolNums; + } + + public int getConsumerProcessorThreadPoolQueueCapacity() { + return consumerProcessorThreadPoolQueueCapacity; + } + + public void setConsumerProcessorThreadPoolQueueCapacity(int consumerProcessorThreadPoolQueueCapacity) { + this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity; + } + + public int getTopicRouteServiceCacheExpiredInSeconds() { + return topicRouteServiceCacheExpiredInSeconds; + } + + public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) { + this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds; + } + + public int getTopicRouteServiceCacheMaxNum() { + return topicRouteServiceCacheMaxNum; + } + + public void setTopicRouteServiceCacheMaxNum(int topicRouteServiceCacheMaxNum) { + this.topicRouteServiceCacheMaxNum = topicRouteServiceCacheMaxNum; + } + + public int getTopicRouteServiceThreadPoolNums() { + return topicRouteServiceThreadPoolNums; + } + + public void setTopicRouteServiceThreadPoolNums(int topicRouteServiceThreadPoolNums) { + this.topicRouteServiceThreadPoolNums = topicRouteServiceThreadPoolNums; + } + + public int getTopicRouteServiceThreadPoolQueueCapacity() { + return topicRouteServiceThreadPoolQueueCapacity; + } + + public void setTopicRouteServiceThreadPoolQueueCapacity(int topicRouteServiceThreadPoolQueueCapacity) { + this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity; + } + + public int getTopicConfigCacheExpiredInSeconds() { + return topicConfigCacheExpiredInSeconds; + } + + public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) { + this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds; + } + + public int getTopicConfigCacheMaxNum() { + return topicConfigCacheMaxNum; + } + + public void setTopicConfigCacheMaxNum(int topicConfigCacheMaxNum) { + this.topicConfigCacheMaxNum = topicConfigCacheMaxNum; + } + + public int getSubscriptionGroupConfigCacheExpiredInSeconds() { + return subscriptionGroupConfigCacheExpiredInSeconds; + } + + public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) { + this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds; + } + + public int getSubscriptionGroupConfigCacheMaxNum() { + return subscriptionGroupConfigCacheMaxNum; + } + + public void setSubscriptionGroupConfigCacheMaxNum(int subscriptionGroupConfigCacheMaxNum) { + this.subscriptionGroupConfigCacheMaxNum = subscriptionGroupConfigCacheMaxNum; + } + + public int getMetadataThreadPoolNums() { + return metadataThreadPoolNums; + } + + public void setMetadataThreadPoolNums(int metadataThreadPoolNums) { + this.metadataThreadPoolNums = metadataThreadPoolNums; + } + + public int getMetadataThreadPoolQueueCapacity() { + return metadataThreadPoolQueueCapacity; + } + + public void setMetadataThreadPoolQueueCapacity(int metadataThreadPoolQueueCapacity) { + this.metadataThreadPoolQueueCapacity = metadataThreadPoolQueueCapacity; + } + + public int getTransactionHeartbeatThreadPoolNums() { + return transactionHeartbeatThreadPoolNums; + } + + public void setTransactionHeartbeatThreadPoolNums(int transactionHeartbeatThreadPoolNums) { + this.transactionHeartbeatThreadPoolNums = transactionHeartbeatThreadPoolNums; + } + + public int getTransactionHeartbeatThreadPoolQueueCapacity() { + return transactionHeartbeatThreadPoolQueueCapacity; + } + + public void setTransactionHeartbeatThreadPoolQueueCapacity(int transactionHeartbeatThreadPoolQueueCapacity) { + this.transactionHeartbeatThreadPoolQueueCapacity = transactionHeartbeatThreadPoolQueueCapacity; + } + + public int getTransactionHeartbeatPeriodSecond() { + return transactionHeartbeatPeriodSecond; + } + + public void setTransactionHeartbeatPeriodSecond(int transactionHeartbeatPeriodSecond) { + this.transactionHeartbeatPeriodSecond = transactionHeartbeatPeriodSecond; + } + + public int getTransactionHeartbeatBatchNum() { + return transactionHeartbeatBatchNum; + } + + public void setTransactionHeartbeatBatchNum(int transactionHeartbeatBatchNum) { + this.transactionHeartbeatBatchNum = transactionHeartbeatBatchNum; + } + + public long getTransactionDataExpireScanPeriodMillis() { + return transactionDataExpireScanPeriodMillis; + } + + public void setTransactionDataExpireScanPeriodMillis(long transactionDataExpireScanPeriodMillis) { + this.transactionDataExpireScanPeriodMillis = transactionDataExpireScanPeriodMillis; + } + + public long getTransactionDataMaxWaitClearMillis() { + return transactionDataMaxWaitClearMillis; + } + + public void setTransactionDataMaxWaitClearMillis(long transactionDataMaxWaitClearMillis) { + this.transactionDataMaxWaitClearMillis = transactionDataMaxWaitClearMillis; + } + + public long getTransactionDataExpireMillis() { + return transactionDataExpireMillis; + } + + public void setTransactionDataExpireMillis(long transactionDataExpireMillis) { + this.transactionDataExpireMillis = transactionDataExpireMillis; + } + + public int getTransactionDataMaxNum() { + return transactionDataMaxNum; + } + + public void setTransactionDataMaxNum(int transactionDataMaxNum) { + this.transactionDataMaxNum = transactionDataMaxNum; + } + + public long getLongPollingReserveTimeInMillis() { + return longPollingReserveTimeInMillis; + } + + public void setLongPollingReserveTimeInMillis(long longPollingReserveTimeInMillis) { + this.longPollingReserveTimeInMillis = longPollingReserveTimeInMillis; + } + + public boolean isEnableACL() { + return enableACL; + } + + public void setEnableACL(boolean enableACL) { + this.enableACL = enableACL; + } + + public boolean isEnableTopicMessageTypeCheck() { + return enableTopicMessageTypeCheck; + } + + public void setEnableTopicMessageTypeCheck(boolean enableTopicMessageTypeCheck) { + this.enableTopicMessageTypeCheck = enableTopicMessageTypeCheck; + } + + public long getInvisibleTimeMillisWhenClear() { + return invisibleTimeMillisWhenClear; + } + + public void setInvisibleTimeMillisWhenClear(long invisibleTimeMillisWhenClear) { + this.invisibleTimeMillisWhenClear = invisibleTimeMillisWhenClear; + } + + public boolean isEnableProxyAutoRenew() { + return enableProxyAutoRenew; + } + + public void setEnableProxyAutoRenew(boolean enableProxyAutoRenew) { + this.enableProxyAutoRenew = enableProxyAutoRenew; + } + + public long getRenewAheadTimeMillis() { + return renewAheadTimeMillis; + } + + public void setRenewAheadTimeMillis(long renewAheadTimeMillis) { + this.renewAheadTimeMillis = renewAheadTimeMillis; + } + + public long getRenewSliceTimeMillis() { + return renewSliceTimeMillis; + } + + public void setRenewSliceTimeMillis(long renewSliceTimeMillis) { + this.renewSliceTimeMillis = renewSliceTimeMillis; + } + + public long getRenewMaxTimeMillis() { + return renewMaxTimeMillis; + } + + public void setRenewMaxTimeMillis(long renewMaxTimeMillis) { + this.renewMaxTimeMillis = renewMaxTimeMillis; + } + + public long getRenewSchedulePeriodMillis() { + return renewSchedulePeriodMillis; + } + + public void setRenewSchedulePeriodMillis(long renewSchedulePeriodMillis) { + this.renewSchedulePeriodMillis = renewSchedulePeriodMillis; + } + + public int getMetricCollectorMode() { + return metricCollectorMode; + } + + public void setMetricCollectorMode(int metricCollectorMode) { + this.metricCollectorMode = metricCollectorMode; + } + + public String getMetricCollectorAddress() { + return metricCollectorAddress; + } + + public void setMetricCollectorAddress(String metricCollectorAddress) { + this.metricCollectorAddress = metricCollectorAddress; + } + + public boolean isUseDelayLevel() { + return useDelayLevel; + } + + public void setUseDelayLevel(boolean useDelayLevel) { + this.useDelayLevel = useDelayLevel; + } + + public String getMessageDelayLevel() { + return messageDelayLevel; + } + + public void setMessageDelayLevel(String messageDelayLevel) { + this.messageDelayLevel = messageDelayLevel; + } + + public Map getDelayLevelTable() { + return delayLevelTable; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java new file mode 100644 index 0000000000..d663a88f6d --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc; + +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.StartAndShutdown; + +public class GrpcServer implements StartAndShutdown { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final io.grpc.Server server; + + protected GrpcServer(io.grpc.Server server) { + this.server = server; + } + + public void start() throws Exception { + this.server.start(); + log.info("grpc server start successfully."); + } + + public void shutdown() { + try { + this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS); + log.info("grpc server shutdown successfully."); + } catch (Exception e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java new file mode 100644 index 0000000000..024766bf48 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc; + +import io.grpc.BindableService; +import io.grpc.ServerInterceptor; +import io.grpc.ServerServiceDefinition; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerSocketChannel; +import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel; +import io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.grpc.netty.shaded.io.netty.handler.ssl.util.SelfSignedCertificate; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.cert.CertificateException; +import java.util.List; +import java.util.concurrent.ThreadPoolExecutor; +import javax.net.ssl.SSLException; +import org.apache.rocketmq.acl.AccessValidator; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.utils.ServiceProvider; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.grpc.interceptor.AuthenticationInterceptor; +import org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor; +import org.apache.rocketmq.proxy.grpc.interceptor.GlobalExceptionInterceptor; +import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor; + +public class GrpcServerBuilder { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected NettyServerBuilder serverBuilder; + + public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) { + return new GrpcServerBuilder(executor, port); + } + + protected GrpcServerBuilder(ThreadPoolExecutor executor, int port) { + serverBuilder = NettyServerBuilder.forPort(port); + + try { + configSslContext(serverBuilder); + } catch (Exception e) { + log.error("grpc tls set failed. msg: {}, e:", e.getMessage(), e); + throw new RuntimeException("grpc tls set failed: " + e.getMessage()); + } + + // build server + int bossLoopNum = ConfigurationManager.getProxyConfig().getGrpcBossLoopNum(); + int workerLoopNum = ConfigurationManager.getProxyConfig().getGrpcWorkerLoopNum(); + int maxInboundMessageSize = ConfigurationManager.getProxyConfig().getGrpcMaxInboundMessageSize(); + + if (ConfigurationManager.getProxyConfig().isEnableGrpcEpoll()) { + serverBuilder.maxInboundMessageSize(maxInboundMessageSize) + .bossEventLoopGroup(new EpollEventLoopGroup(bossLoopNum)) + .workerEventLoopGroup(new EpollEventLoopGroup(workerLoopNum)) + .channelType(EpollServerSocketChannel.class) + .executor(executor); + } else { + serverBuilder.maxInboundMessageSize(maxInboundMessageSize) + .bossEventLoopGroup(new NioEventLoopGroup(bossLoopNum)) + .workerEventLoopGroup(new NioEventLoopGroup(workerLoopNum)) + .channelType(NioServerSocketChannel.class) + .executor(executor); + } + + log.info( + "grpc server has built. port: {}, tlsKeyPath: {}, tlsCertPath: {}, threadPool: {}, queueCapacity: {}, " + + "boosLoop: {}, workerLoop: {}, maxInboundMessageSize: {}", + port, bossLoopNum, workerLoopNum, maxInboundMessageSize); + } + + public GrpcServerBuilder addService(BindableService service) { + this.serverBuilder.addService(service); + return this; + } + + public GrpcServerBuilder addService(ServerServiceDefinition service) { + this.serverBuilder.addService(service); + return this; + } + + public GrpcServerBuilder appendInterceptor(ServerInterceptor interceptor) { + this.serverBuilder.intercept(interceptor); + return this; + } + + public GrpcServer build() { + return new GrpcServer(this.serverBuilder.build()); + } + + protected void configSslContext(NettyServerBuilder serverBuilder) throws SSLException, CertificateException { + if (null == serverBuilder) { + return; + } + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + boolean tlsTestModeEnable = proxyConfig.isGrpcTlsTestModeEnable(); + if (tlsTestModeEnable) { + SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate(); + serverBuilder.sslContext(GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey()) + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .clientAuth(ClientAuth.NONE) + .build()); + return; + } + + String tlsKeyPath = ConfigurationManager.getProxyConfig().getGrpcTlsKeyPath(); + String tlsCertPath = ConfigurationManager.getProxyConfig().getGrpcTlsCertPath(); + try (InputStream serverKeyInputStream = Files.newInputStream(Paths.get(tlsKeyPath)); + InputStream serverCertificateStream = Files.newInputStream(Paths.get(tlsCertPath))) { + serverBuilder.sslContext(GrpcSslContexts.forServer(serverCertificateStream, serverKeyInputStream) + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .clientAuth(ClientAuth.NONE) + .build()); + log.info("TLS configured OK"); + } catch (IOException e) { + log.error("Failed to load Server key/certificate", e); + } + } + + public GrpcServerBuilder configInterceptor() { + // grpc interceptors, including acl, logging etc. + List accessValidators = ServiceProvider.load(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class); + if (!accessValidators.isEmpty()) { + this.serverBuilder.intercept(new AuthenticationInterceptor(accessValidators)); + } + + this.serverBuilder + .intercept(new GlobalExceptionInterceptor()) + .intercept(new ContextInterceptor()) + .intercept(new HeaderInterceptor()); + + return this; + } + +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java new file mode 100644 index 0000000000..5aa009e733 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/AuthenticationInterceptor.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import com.google.protobuf.GeneratedMessageV3; +import io.grpc.Context; +import io.grpc.ForwardingServerCallListener; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import java.util.List; +import org.apache.rocketmq.acl.AccessResource; +import org.apache.rocketmq.acl.AccessValidator; +import org.apache.rocketmq.acl.common.AclException; +import org.apache.rocketmq.acl.common.AuthenticationHeader; +import org.apache.rocketmq.acl.plain.PlainAccessResource; +import org.apache.rocketmq.proxy.config.ConfigurationManager; + +public class AuthenticationInterceptor implements ServerInterceptor { + protected final List accessValidatorList; + + public AuthenticationInterceptor(List accessValidatorList) { + this.accessValidatorList = accessValidatorList; + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, Metadata headers, + ServerCallHandler next) { + return new ForwardingServerCallListener.SimpleForwardingServerCallListener(next.startCall(call, headers)) { + @Override + public void onMessage(R message) { + GeneratedMessageV3 messageV3 = (GeneratedMessageV3) message; + headers.put(InterceptorConstants.RPC_NAME, messageV3.getDescriptorForType().getFullName()); + if (ConfigurationManager.getProxyConfig().isEnableACL()) { + try { + AuthenticationHeader authenticationHeader = AuthenticationHeader.builder() + .remoteAddress(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.REMOTE_ADDRESS)) + .namespace(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.NAMESPACE_ID)) + .authorization(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.AUTHORIZATION)) + .datetime(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.DATE_TIME)) + .sessionToken(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.SESSION_TOKEN)) + .requestId(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.REQUEST_ID)) + .language(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.LANGUAGE)) + .clientVersion(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.CLIENT_VERSION)) + .protocol(InterceptorConstants.METADATA.get(Context.current()).get(InterceptorConstants.PROTOCOL_VERSION)) + .requestCode(RequestMapping.map(messageV3.getDescriptorForType().getFullName())) + .build(); + + validate(authenticationHeader, headers, messageV3); + super.onMessage(message); + } catch (AclException aclException) { + throw new StatusRuntimeException(Status.PERMISSION_DENIED, headers); + } + } else { + super.onMessage(message); + } + } + }; + } + + protected void validate(AuthenticationHeader authenticationHeader, Metadata headers, GeneratedMessageV3 messageV3) { + for (AccessValidator accessValidator : accessValidatorList) { + AccessResource accessResource = accessValidator.parse(messageV3, authenticationHeader); + accessValidator.validate(accessResource); + + if (accessResource instanceof PlainAccessResource) { + PlainAccessResource plainAccessResource = (PlainAccessResource) accessResource; + headers.put(InterceptorConstants.AUTHORIZATION_AK, plainAccessResource.getAccessKey()); + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java new file mode 100644 index 0000000000..07d7ab9bf3 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/ContextInterceptor.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; + +public class ContextInterceptor implements ServerInterceptor { + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next + ) { + Context context = Context.current().withValue(InterceptorConstants.METADATA, headers); + return Contexts.interceptCall(context, call, headers, next); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/GlobalExceptionInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/GlobalExceptionInterceptor.java new file mode 100644 index 0000000000..0c34b15743 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/GlobalExceptionInterceptor.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import io.grpc.ForwardingServerCall; +import io.grpc.ForwardingServerCallListener; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class GlobalExceptionInterceptor implements ServerInterceptor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next + ) { + final ServerCall serverCall = new ClosableServerCall<>(call); + ServerCall.Listener delegate = next.startCall(serverCall, headers); + return new ForwardingServerCallListener.SimpleForwardingServerCallListener(delegate) { + @Override + public void onMessage(R message) { + try { + super.onMessage(message); + } catch (Throwable e) { + closeWithException(e); + } + } + + @Override + public void onHalfClose() { + try { + super.onHalfClose(); + } catch (Throwable e) { + closeWithException(e); + } + } + + @Override + public void onCancel() { + try { + super.onCancel(); + } catch (Throwable e) { + closeWithException(e); + } + } + + @Override + public void onComplete() { + try { + super.onComplete(); + } catch (Throwable e) { + closeWithException(e); + } + } + + @Override + public void onReady() { + try { + super.onReady(); + } catch (Throwable e) { + closeWithException(e); + } + } + + private void closeWithException(Throwable t) { + Metadata trailers = new Metadata(); + Status status = Status.INTERNAL.withDescription(t.getMessage()); + boolean printLog = true; + + if (t instanceof StatusRuntimeException) { + trailers = ((StatusRuntimeException) t).getTrailers(); + status = ((StatusRuntimeException) t).getStatus(); + // no error stack for permission denied. + if (status.getCode().value() == Status.PERMISSION_DENIED.getCode().value()) { + printLog = false; + } + } + + if (printLog) { + log.error("grpc server has exception. errorMsg:{}, e:", t.getMessage(), t); + } + + serverCall.close(status, trailers); + } + }; + } + + private static class ClosableServerCall extends + ForwardingServerCall.SimpleForwardingServerCall { + private boolean closeCalled = false; + + ClosableServerCall(ServerCall delegate) { + super(delegate); + } + + @Override + public synchronized void close(final Status status, final Metadata trailers) { + if (!closeCalled) { + closeCalled = true; + ClosableServerCall.super.close(status, trailers); + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java new file mode 100644 index 0000000000..1cbb003610 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/HeaderInterceptor.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import com.google.common.net.HostAndPort; +import io.grpc.Grpc; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import java.net.InetSocketAddress; +import java.net.SocketAddress; + +public class HeaderInterceptor implements ServerInterceptor { + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next + ) { + SocketAddress remoteSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR); + String remoteAddress = parseSocketAddress(remoteSocketAddress); + headers.put(InterceptorConstants.REMOTE_ADDRESS, remoteAddress); + + SocketAddress localSocketAddress = call.getAttributes().get(Grpc.TRANSPORT_ATTR_LOCAL_ADDR); + String localAddress = parseSocketAddress(localSocketAddress); + headers.put(InterceptorConstants.LOCAL_ADDRESS, localAddress); + return next.startCall(call, headers); + } + + private String parseSocketAddress(SocketAddress socketAddress) { + if (socketAddress instanceof InetSocketAddress) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + return HostAndPort.fromParts( + inetSocketAddress.getAddress() + .getHostAddress(), + inetSocketAddress.getPort() + ).toString(); + } + + return ""; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java new file mode 100644 index 0000000000..c8aa39959e --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/InterceptorConstants.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import io.grpc.Context; +import io.grpc.Metadata; + +public class InterceptorConstants { + public static final Context.Key METADATA = Context.key("rpc-metadata"); + + /** + * Remote address key in attributes of call + */ + public static final Metadata.Key REMOTE_ADDRESS + = Metadata.Key.of("rpc-remote-address", Metadata.ASCII_STRING_MARSHALLER); + + /** + * Local address key in attributes of call + */ + public static final Metadata.Key LOCAL_ADDRESS + = Metadata.Key.of("rpc-local-address", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key AUTHORIZATION + = Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key NAMESPACE_ID + = Metadata.Key.of("x-mq-namespace", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key DATE_TIME + = Metadata.Key.of("x-mq-date-time", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key REQUEST_ID + = Metadata.Key.of("x-mq-request-id", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key LANGUAGE + = Metadata.Key.of("x-mq-language", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key CLIENT_VERSION + = Metadata.Key.of("x-mq-client-version", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key PROTOCOL_VERSION + = Metadata.Key.of("x-mq-protocol", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key RPC_NAME + = Metadata.Key.of("x-mq-rpc-name", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key SESSION_TOKEN + = Metadata.Key.of("x-mq-session-token", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key CLIENT_ID + = Metadata.Key.of("x-mq-client-id", Metadata.ASCII_STRING_MARSHALLER); + + public static final Metadata.Key AUTHORIZATION_AK + = Metadata.Key.of("x-mq-authorization-ak", Metadata.ASCII_STRING_MARSHALLER); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java new file mode 100644 index 0000000000..a9674d1837 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/interceptor/RequestMapping.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.interceptor; + +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.SendMessageRequest; +import java.util.HashMap; +import java.util.Map; +import org.apache.rocketmq.common.protocol.RequestCode; + +public class RequestMapping { + private final static Map REQUEST_MAP = new HashMap() { + { + // v2 + put(QueryRouteRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC); + put(HeartbeatRequest.getDescriptor().getFullName(), RequestCode.HEART_BEAT); + put(SendMessageRequest.getDescriptor().getFullName(), RequestCode.SEND_MESSAGE_V2); + put(QueryAssignmentRequest.getDescriptor().getFullName(), RequestCode.GET_ROUTEINFO_BY_TOPIC); + put(ReceiveMessageRequest.getDescriptor().getFullName(), RequestCode.PULL_MESSAGE); + put(AckMessageRequest.getDescriptor().getFullName(), RequestCode.UPDATE_CONSUMER_OFFSET); + put(ForwardMessageToDeadLetterQueueResponse.getDescriptor().getFullName(), RequestCode.CONSUMER_SEND_MSG_BACK); + put(EndTransactionRequest.getDescriptor().getFullName(), RequestCode.END_TRANSACTION); + put(NotifyClientTerminationRequest.getDescriptor().getFullName(), RequestCode.UNREGISTER_CLIENT); + put(ChangeInvisibleDurationRequest.getDescriptor().getFullName(), RequestCode.CONSUMER_SEND_MSG_BACK); + } + }; + + public static int map(String rpcFullName) { + if (REQUEST_MAP.containsKey(rpcFullName)) { + return REQUEST_MAP.get(rpcFullName); + } + return RequestCode.HEART_BEAT; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivity.java new file mode 100644 index 0000000000..13b855768a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivity.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.Resource; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcValidator; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; + +public abstract class AbstractMessingActivity { + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected final MessagingProcessor messagingProcessor; + protected final GrpcClientSettingsManager grpcClientSettingsManager; + protected final GrpcChannelManager grpcChannelManager; + + public AbstractMessingActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + this.messagingProcessor = messagingProcessor; + this.grpcClientSettingsManager = grpcClientSettingsManager; + this.grpcChannelManager = grpcChannelManager; + } + + protected void validateTopic(Resource topic) { + GrpcValidator.getInstance().validateTopic(topic); + } + + protected void validateConsumerGroup(Resource consumerGroup) { + GrpcValidator.getInstance().validateConsumerGroup(consumerGroup); + } + + protected void validateTopicAndConsumerGroup(Resource topic, Resource consumerGroup) { + GrpcValidator.getInstance().validateTopicAndConsumerGroup(topic, consumerGroup); + } + + protected void validateInvisibleTime(long invisibleTime) { + GrpcValidator.getInstance().validateInvisibleTime(invisibleTime); + } + + protected void validateInvisibleTime(long invisibleTime, long minInvisibleTime) { + GrpcValidator.getInstance().validateInvisibleTime(invisibleTime, minInvisibleTime); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java new file mode 100644 index 0000000000..8cac746bf6 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/DefaultGrpcMessingActivity.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.HeartbeatResponse; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.TelemetryCommand; +import io.grpc.stub.StreamObserver; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.client.ClientActivity; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.consumer.AckMessageActivity; +import org.apache.rocketmq.proxy.grpc.v2.consumer.ChangeInvisibleDurationActivity; +import org.apache.rocketmq.proxy.grpc.v2.consumer.ReceiveMessageActivity; +import org.apache.rocketmq.proxy.grpc.v2.producer.ForwardMessageToDLQActivity; +import org.apache.rocketmq.proxy.grpc.v2.producer.SendMessageActivity; +import org.apache.rocketmq.proxy.grpc.v2.route.RouteActivity; +import org.apache.rocketmq.proxy.grpc.v2.transaction.EndTransactionActivity; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor; + +public class DefaultGrpcMessingActivity extends AbstractStartAndShutdown implements GrpcMessingActivity { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected GrpcClientSettingsManager grpcClientSettingsManager; + protected GrpcChannelManager grpcChannelManager; + protected ReceiptHandleProcessor receiptHandleProcessor; + protected ReceiveMessageActivity receiveMessageActivity; + protected AckMessageActivity ackMessageActivity; + protected ChangeInvisibleDurationActivity changeInvisibleDurationActivity; + protected SendMessageActivity sendMessageActivity; + protected ForwardMessageToDLQActivity forwardMessageToDLQActivity; + protected EndTransactionActivity endTransactionActivity; + protected RouteActivity routeActivity; + protected ClientActivity clientActivity; + + protected DefaultGrpcMessingActivity(MessagingProcessor messagingProcessor) { + this.grpcClientSettingsManager = new GrpcClientSettingsManager(messagingProcessor); + this.grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService()); + this.receiptHandleProcessor = new ReceiptHandleProcessor(messagingProcessor); + + this.receiveMessageActivity = new ReceiveMessageActivity(messagingProcessor, receiptHandleProcessor, grpcClientSettingsManager, grpcChannelManager); + this.ackMessageActivity = new AckMessageActivity(messagingProcessor, receiptHandleProcessor, grpcClientSettingsManager, grpcChannelManager); + this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor, receiptHandleProcessor, grpcClientSettingsManager, grpcChannelManager); + this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.clientActivity = new ClientActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + + this.init(); + } + + protected void init() { + this.appendStartAndShutdown(this.receiptHandleProcessor); + } + + @Override + public CompletableFuture queryRoute(ProxyContext ctx, QueryRouteRequest request) { + return this.routeActivity.queryRoute(ctx, request); + } + + @Override + public CompletableFuture heartbeat(ProxyContext ctx, HeartbeatRequest request) { + return this.clientActivity.heartbeat(ctx, request); + } + + @Override + public CompletableFuture sendMessage(ProxyContext ctx, SendMessageRequest request) { + return this.sendMessageActivity.sendMessage(ctx, request); + } + + @Override + public CompletableFuture queryAssignment(ProxyContext ctx, + QueryAssignmentRequest request) { + return this.routeActivity.queryAssignment(ctx, request); + } + + @Override + public void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request, + StreamObserver responseObserver) { + this.receiveMessageActivity.receiveMessage(ctx, request, responseObserver); + } + + @Override + public CompletableFuture ackMessage(ProxyContext ctx, AckMessageRequest request) { + return this.ackMessageActivity.ackMessage(ctx, request); + } + + @Override + public CompletableFuture forwardMessageToDeadLetterQueue(ProxyContext ctx, + ForwardMessageToDeadLetterQueueRequest request) { + return this.forwardMessageToDLQActivity.forwardMessageToDeadLetterQueue(ctx, request); + } + + @Override + public CompletableFuture endTransaction(ProxyContext ctx, EndTransactionRequest request) { + return this.endTransactionActivity.endTransaction(ctx, request); + } + + @Override + public CompletableFuture notifyClientTermination(ProxyContext ctx, + NotifyClientTerminationRequest request) { + return this.clientActivity.notifyClientTermination(ctx, request); + } + + @Override + public CompletableFuture changeInvisibleDuration(ProxyContext ctx, + ChangeInvisibleDurationRequest request) { + return this.changeInvisibleDurationActivity.changeInvisibleDuration(ctx, request); + } + + @Override + public StreamObserver telemetry(ProxyContext ctx, + StreamObserver responseObserver) { + return this.clientActivity.telemetry(ctx, responseObserver); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java new file mode 100644 index 0000000000..9c940dee76 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplication.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.HeartbeatResponse; +import apache.rocketmq.v2.MessagingServiceGrpc; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.Status; +import apache.rocketmq.v2.TelemetryCommand; +import io.grpc.Context; +import io.grpc.Metadata; +import io.grpc.stub.StreamObserver; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseWriter; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GrpcMessagingApplication extends MessagingServiceGrpc.MessagingServiceImplBase implements StartAndShutdown { + private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final GrpcMessingActivity grpcMessingActivity; + + protected ThreadPoolExecutor routeThreadPoolExecutor; + protected ThreadPoolExecutor producerThreadPoolExecutor; + protected ThreadPoolExecutor consumerThreadPoolExecutor; + protected ThreadPoolExecutor clientManagerThreadPoolExecutor; + protected ThreadPoolExecutor transactionThreadPoolExecutor; + + protected GrpcMessagingApplication(GrpcMessingActivity grpcMessingActivity) { + this.grpcMessingActivity = grpcMessingActivity; + + ProxyConfig config = ConfigurationManager.getProxyConfig(); + this.routeThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor( + config.getGrpcRouteThreadPoolNums(), + config.getGrpcRouteThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "GrpcRouteThreadPool", + config.getGrpcRouteThreadQueueCapacity() + ); + this.producerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor( + config.getGrpcProducerThreadPoolNums(), + config.getGrpcProducerThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "GrpcProducerThreadPool", + config.getGrpcProducerThreadQueueCapacity() + ); + this.consumerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor( + config.getGrpcConsumerThreadPoolNums(), + config.getGrpcConsumerThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "GrpcConsumerThreadPool", + config.getGrpcConsumerThreadQueueCapacity() + ); + this.clientManagerThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor( + config.getGrpcClientManagerThreadPoolNums(), + config.getGrpcClientManagerThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "GrpcClientManagerThreadPool", + config.getGrpcClientManagerThreadQueueCapacity() + ); + this.transactionThreadPoolExecutor = ThreadPoolMonitor.createAndMonitor( + config.getGrpcTransactionThreadPoolNums(), + config.getGrpcTransactionThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "GrpcTransactionThreadPool", + config.getGrpcTransactionThreadQueueCapacity() + ); + + this.init(); + } + + protected void init() { + GrpcTaskRejectedExecutionHandler rejectedExecutionHandler = new GrpcTaskRejectedExecutionHandler(); + this.routeThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + this.routeThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + this.producerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + this.consumerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + this.clientManagerThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + this.transactionThreadPoolExecutor.setRejectedExecutionHandler(rejectedExecutionHandler); + } + + public static GrpcMessagingApplication create(MessagingProcessor messagingProcessor) { + return new GrpcMessagingApplication(new DefaultGrpcMessingActivity( + messagingProcessor + )); + } + + protected Status flowLimitStatus() { + return ResponseBuilder.getInstance().buildStatus(Code.TOO_MANY_REQUESTS, "flow limit"); + } + + protected Status convertExceptionToStatus(Throwable t) { + return ResponseBuilder.getInstance().buildStatus(t); + } + + protected void addExecutor(ExecutorService executor, ProxyContext context, V request, Runnable runnable, + StreamObserver responseObserver, Function statusResponseCreator) { + executor.submit(new GrpcTask<>(runnable, context, request, responseObserver, statusResponseCreator.apply(flowLimitStatus()))); + } + + protected void writeResponse(ProxyContext context, V request, T response, StreamObserver responseObserver, + Throwable t, Function errorResponseCreator) { + if (t != null) { + ResponseWriter.getInstance().write( + responseObserver, + errorResponseCreator.apply(convertExceptionToStatus(t)) + ); + } else { + ResponseWriter.getInstance().write(responseObserver, response); + } + } + + protected ProxyContext createContext() { + Context ctx = Context.current(); + Metadata headers = InterceptorConstants.METADATA.get(ctx); + ProxyContext context = ProxyContext.create() + .setLocalAddress(getDefaultStringMetadataInfo(headers, InterceptorConstants.LOCAL_ADDRESS)) + .setRemoteAddress(getDefaultStringMetadataInfo(headers, InterceptorConstants.REMOTE_ADDRESS)) + .setClientID(getDefaultStringMetadataInfo(headers, InterceptorConstants.CLIENT_ID)) + .setLanguage(getDefaultStringMetadataInfo(headers, InterceptorConstants.LANGUAGE)) + .setClientVersion(getDefaultStringMetadataInfo(headers, InterceptorConstants.CLIENT_VERSION)) + .setAction(getDefaultStringMetadataInfo(headers, InterceptorConstants.RPC_NAME)); + if (ctx.getDeadline() != null) { + context.setRemainingMs(ctx.getDeadline().timeRemaining(TimeUnit.MILLISECONDS)); + } + return context; + } + + protected void validateContext(ProxyContext context) { + if (StringUtils.isBlank(context.getClientID())) { + throw new GrpcProxyException(Code.CLIENT_ID_REQUIRED, "client id cannot be empty"); + } + } + + protected String getDefaultStringMetadataInfo(Metadata headers, Metadata.Key key) { + return StringUtils.defaultString(headers.get(key)); + } + + @Override + public void queryRoute(QueryRouteRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> QueryRouteResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.routeThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.queryRoute(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void heartbeat(HeartbeatRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> HeartbeatResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.clientManagerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.heartbeat(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void sendMessage(SendMessageRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> SendMessageResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.producerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.sendMessage(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void queryAssignment(QueryAssignmentRequest request, + StreamObserver responseObserver) { + Function statusResponseCreator = status -> QueryAssignmentResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.routeThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.queryAssignment(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void receiveMessage(ReceiveMessageRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> ReceiveMessageResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.consumerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.receiveMessage(context, request, responseObserver), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void ackMessage(AckMessageRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> AckMessageResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.consumerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.ackMessage(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void forwardMessageToDeadLetterQueue(ForwardMessageToDeadLetterQueueRequest request, + StreamObserver responseObserver) { + Function statusResponseCreator = status -> ForwardMessageToDeadLetterQueueResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.producerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.forwardMessageToDeadLetterQueue(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void endTransaction(EndTransactionRequest request, StreamObserver responseObserver) { + Function statusResponseCreator = status -> EndTransactionResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.transactionThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.endTransaction(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void notifyClientTermination(NotifyClientTerminationRequest request, + StreamObserver responseObserver) { + Function statusResponseCreator = status -> NotifyClientTerminationResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.clientManagerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.notifyClientTermination(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void changeInvisibleDuration(ChangeInvisibleDurationRequest request, + StreamObserver responseObserver) { + Function statusResponseCreator = status -> ChangeInvisibleDurationResponse.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + try { + validateContext(context); + this.addExecutor(this.consumerThreadPoolExecutor, + context, + request, + () -> grpcMessingActivity.changeInvisibleDuration(context, request) + .whenComplete((response, throwable) -> writeResponse(context, request, response, responseObserver, throwable, statusResponseCreator)), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, request, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public StreamObserver telemetry(StreamObserver responseObserver) { + Function statusResponseCreator = status -> TelemetryCommand.newBuilder().setStatus(status).build(); + ProxyContext context = createContext(); + StreamObserver responseTelemetryCommand = grpcMessingActivity.telemetry(context, responseObserver); + return new StreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + try { + validateContext(context); + addExecutor(clientManagerThreadPoolExecutor, + context, + value, + () -> responseTelemetryCommand.onNext(value), + responseObserver, + statusResponseCreator); + } catch (Throwable t) { + writeResponse(context, value, null, responseObserver, t, statusResponseCreator); + } + } + + @Override + public void onError(Throwable t) { + responseTelemetryCommand.onError(t); + } + + @Override + public void onCompleted() { + responseTelemetryCommand.onCompleted(); + } + }; + } + + @Override + public void shutdown() throws Exception { + this.grpcMessingActivity.shutdown(); + + this.routeThreadPoolExecutor.shutdown(); + this.routeThreadPoolExecutor.shutdown(); + this.producerThreadPoolExecutor.shutdown(); + this.consumerThreadPoolExecutor.shutdown(); + this.clientManagerThreadPoolExecutor.shutdown(); + this.transactionThreadPoolExecutor.shutdown(); + } + + @Override + public void start() throws Exception { + this.grpcMessingActivity.start(); + } + + protected static class GrpcTask implements Runnable { + + protected final Runnable runnable; + protected final ProxyContext context; + protected final V request; + protected final T executeRejectResponse; + protected final StreamObserver streamObserver; + + public GrpcTask(Runnable runnable, ProxyContext context, V request, StreamObserver streamObserver, + T executeRejectResponse) { + this.runnable = runnable; + this.context = context; + this.streamObserver = streamObserver; + this.request = request; + this.executeRejectResponse = executeRejectResponse; + } + + @Override + public void run() { + this.runnable.run(); + } + } + + protected class GrpcTaskRejectedExecutionHandler implements RejectedExecutionHandler { + + public GrpcTaskRejectedExecutionHandler() { + + } + + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + if (r instanceof GrpcTask) { + try { + GrpcTask grpcTask = (GrpcTask) r; + writeResponse(grpcTask.context, grpcTask.request, grpcTask.executeRejectResponse, grpcTask.streamObserver, null, null); + } catch (Throwable t) { + log.warn("write rejected error response failed", t); + } + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java new file mode 100644 index 0000000000..0f353e94db --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessingActivity.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.HeartbeatResponse; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.TelemetryCommand; +import io.grpc.stub.StreamObserver; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.StartAndShutdown; + +public interface GrpcMessingActivity extends StartAndShutdown { + + CompletableFuture queryRoute(ProxyContext ctx, QueryRouteRequest request); + + CompletableFuture heartbeat(ProxyContext ctx, HeartbeatRequest request); + + CompletableFuture sendMessage(ProxyContext ctx, SendMessageRequest request); + + CompletableFuture queryAssignment(ProxyContext ctx, QueryAssignmentRequest request); + + void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request, + StreamObserver responseObserver); + + CompletableFuture ackMessage(ProxyContext ctx, AckMessageRequest request); + + CompletableFuture forwardMessageToDeadLetterQueue(ProxyContext ctx, + ForwardMessageToDeadLetterQueueRequest request); + + CompletableFuture endTransaction(ProxyContext ctx, EndTransactionRequest request); + + CompletableFuture notifyClientTermination(ProxyContext ctx, + NotifyClientTerminationRequest request); + + CompletableFuture changeInvisibleDuration(ProxyContext ctx, + ChangeInvisibleDurationRequest request); + + StreamObserver telemetry(ProxyContext ctx, StreamObserver responseObserver); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java new file mode 100644 index 0000000000..57a7b1104b --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.channel; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; + +public class GrpcChannelManager implements StartAndShutdown { + private final ProxyRelayService proxyRelayService; + protected final ConcurrentMap/* clientId */> groupClientIdChannelMap = new ConcurrentHashMap<>(); + + protected final AtomicLong nonceIdGenerator = new AtomicLong(0); + protected final ConcurrentMap resultNonceFutureMap = new ConcurrentHashMap<>(); + + protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("GrpcChannelManager_") + ); + + public GrpcChannelManager(ProxyRelayService proxyRelayService) { + this.proxyRelayService = proxyRelayService; + } + + protected void init() { + this.scheduledExecutorService.scheduleAtFixedRate( + this::scanExpireResultFuture, + 10, 10, TimeUnit.SECONDS + ); + } + + public GrpcClientChannel createChannel(ProxyContext ctx, String group, String clientId) { + this.groupClientIdChannelMap.compute(group, (groupKey, clientIdMap) -> { + if (clientIdMap == null) { + clientIdMap = new ConcurrentHashMap<>(); + } + clientIdMap.computeIfAbsent(clientId, clientIdKey -> new GrpcClientChannel(proxyRelayService, this, ctx, group, clientId)); + return clientIdMap; + }); + return getChannel(group, clientId); + } + + public GrpcClientChannel getChannel(String group, String clientId) { + Map clientIdChannelMap = this.groupClientIdChannelMap.get(group); + if (clientIdChannelMap == null) { + return null; + } + return clientIdChannelMap.get(clientId); + } + + public GrpcClientChannel removeChannel(String group, String clientId) { + AtomicReference channelRef = new AtomicReference<>(); + this.groupClientIdChannelMap.computeIfPresent(group, (groupKey, clientIdMap) -> { + channelRef.set(clientIdMap.remove(clientId)); + if (clientIdMap.isEmpty()) { + return null; + } + return clientIdMap; + }); + return channelRef.get(); + } + + public String addResponseFuture(CompletableFuture> responseFuture) { + String nonce = this.nextNonce(); + this.resultNonceFutureMap.put(nonce, new ResultFuture<>(responseFuture)); + return nonce; + } + + public CompletableFuture> getAndRemoveResponseFuture(String nonce) { + ResultFuture resultFuture = this.resultNonceFutureMap.remove(nonce); + if (resultFuture != null) { + return resultFuture.future; + } + return null; + } + + protected String nextNonce() { + return String.valueOf(this.nonceIdGenerator.getAndIncrement()); + } + + protected void scanExpireResultFuture() { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + long timeOutMs = TimeUnit.SECONDS.toMillis(proxyConfig.getGrpcProxyRelayRequestTimeoutInSeconds()); + + Set nonceSet = this.resultNonceFutureMap.keySet(); + for (String nonce : nonceSet) { + ResultFuture resultFuture = this.resultNonceFutureMap.get(nonce); + if (resultFuture == null) { + continue; + } + if (System.currentTimeMillis() - resultFuture.createTime > timeOutMs) { + resultFuture = this.resultNonceFutureMap.remove(nonce); + if (resultFuture != null) { + resultFuture.future.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_BUSY, "call remote timeout", null)); + } + } + } + } + + @Override + public void shutdown() throws Exception { + this.scheduledExecutorService.shutdown(); + } + + @Override + public void start() throws Exception { + + } + + protected static class ResultFuture { + public CompletableFuture> future; + public long createTime = System.currentTimeMillis(); + + public ResultFuture(CompletableFuture> future) { + this.future = future; + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java new file mode 100644 index 0000000000..d0ef56159a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcClientChannel.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.channel; + +import apache.rocketmq.v2.PrintThreadStackTraceCommand; +import apache.rocketmq.v2.RecoverOrphanedTransactionCommand; +import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.VerifyMessageCommand; +import com.google.common.collect.ComparisonChain; +import io.grpc.stub.StreamObserver; +import io.netty.channel.ChannelId; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.service.relay.ProxyChannel; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class GrpcClientChannel extends ProxyChannel { + + protected static final String SEPARATOR = "@"; + + private final GrpcChannelManager grpcChannelManager; + + private final AtomicReference> telemetryCommandRef = new AtomicReference<>(); + private final String group; + private final String clientId; + + public GrpcClientChannel(ProxyRelayService proxyRelayService, GrpcChannelManager grpcChannelManager, + ProxyContext ctx, + String group, String clientId) { + super(proxyRelayService, null, new GrpcChannelId(group, clientId), + ctx.getRemoteAddress(), + ctx.getLocalAddress()); + this.grpcChannelManager = grpcChannelManager; + this.group = group; + this.clientId = clientId; + } + + protected static class GrpcChannelId implements ChannelId { + + private final String group; + private final String clientId; + + public GrpcChannelId(String group, String clientId) { + this.group = group; + this.clientId = clientId; + } + + @Override + public String asShortText() { + return this.clientId; + } + + @Override + public String asLongText() { + return this.group + SEPARATOR + this.clientId; + } + + @Override + public int compareTo(ChannelId o) { + if (this == o) { + return 0; + } + if (o instanceof GrpcChannelId) { + GrpcChannelId other = (GrpcChannelId) o; + return ComparisonChain.start() + .compare(this.group, other.group) + .compare(this.clientId, other.clientId) + .result(); + } + + return asLongText().compareTo(o.asLongText()); + } + } + + public void setClientObserver(StreamObserver future) { + this.telemetryCommandRef.set(future); + } + + @Override + public boolean isOpen() { + return this.telemetryCommandRef.get() != null; + } + + @Override + public boolean isActive() { + return this.telemetryCommandRef.get() != null; + } + + @Override + public boolean isWritable() { + return this.telemetryCommandRef.get() != null; + } + + @Override + protected CompletableFuture processOtherMessage(Object msg) { + if (msg instanceof TelemetryCommand) { + TelemetryCommand response = (TelemetryCommand) msg; + this.getTelemetryCommandStreamObserver().onNext(response); + } + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture processCheckTransaction(CheckTransactionStateRequestHeader header, + MessageExt messageExt, TransactionData transactionData, CompletableFuture> responseFuture) { + CompletableFuture writeFuture = new CompletableFuture<>(); + try { + this.getTelemetryCommandStreamObserver().onNext(TelemetryCommand.newBuilder() + .setRecoverOrphanedTransactionCommand(RecoverOrphanedTransactionCommand.newBuilder() + .setTransactionId(transactionData.getTransactionId()) + .setMessage(GrpcConverter.getInstance().buildMessage(messageExt)) + .build()) + .build()); + responseFuture.complete(null); + writeFuture.complete(null); + } catch (Throwable t) { + responseFuture.completeExceptionally(t); + writeFuture.completeExceptionally(t); + } + return writeFuture; + } + + @Override + protected CompletableFuture processGetConsumerRunningInfo(RemotingCommand command, + GetConsumerRunningInfoRequestHeader header, + CompletableFuture> responseFuture) { + if (!header.isJstackEnable()) { + return CompletableFuture.completedFuture(null); + } + this.getTelemetryCommandStreamObserver().onNext(TelemetryCommand.newBuilder() + .setPrintThreadStackTraceCommand(PrintThreadStackTraceCommand.newBuilder() + .setNonce(this.grpcChannelManager.addResponseFuture(responseFuture)) + .build()) + .build()); + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture processConsumeMessageDirectly(RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header, + MessageExt messageExt, CompletableFuture> responseFuture) { + this.getTelemetryCommandStreamObserver().onNext(TelemetryCommand.newBuilder() + .setVerifyMessageCommand(VerifyMessageCommand.newBuilder() + .setNonce(this.grpcChannelManager.addResponseFuture(responseFuture)) + .setMessage(GrpcConverter.getInstance().buildMessage(messageExt)) + .build()) + .build()); + return CompletableFuture.completedFuture(null); + } + + public String getGroup() { + return group; + } + + public String getClientId() { + return clientId; + } + + public String getRemoteAddress() { + return remoteAddress; + } + + public String getLocalAddress() { + return localAddress; + } + + public StreamObserver getTelemetryCommandStreamObserver() { + return this.telemetryCommandRef.get(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java new file mode 100644 index 0000000000..1f58e70661 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivity.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.client; + +import apache.rocketmq.v2.ClientType; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.HeartbeatResponse; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Status; +import apache.rocketmq.v2.SubscriptionEntry; +import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.ThreadStackTrace; +import apache.rocketmq.v2.VerifyMessageResult; +import io.grpc.stub.StreamObserver; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupEvent; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.broker.client.ProducerChangeListener; +import org.apache.rocketmq.broker.client.ProducerGroupEvent; +import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.CMResult; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.remoting.protocol.LanguageCode; + +public class ClientActivity extends AbstractMessingActivity { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + public ClientActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, + GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.init(); + } + + protected void init() { + this.messagingProcessor.registerConsumerListener(new ConsumerIdsChangeListenerImpl()); + this.messagingProcessor.registerProducerListener(new ProducerChangeListenerImpl()); + } + + public CompletableFuture heartbeat(ProxyContext ctx, HeartbeatRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + Settings clientSettings = grpcClientSettingsManager.getClientSettings(ctx); + if (clientSettings == null) { + future.complete(HeartbeatResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, "cannot find client settings for this client")) + .build()); + return future; + } + switch (clientSettings.getClientType()) { + case PRODUCER: { + for (Resource topic : clientSettings.getPublishing().getTopicsList()) { + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(topic); + this.registerProducer(ctx, topicName); + } + break; + } + case PUSH_CONSUMER: + case SIMPLE_CONSUMER: { + validateConsumerGroup(request.getGroup()); + String consumerGroup = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + this.registerConsumer(ctx, consumerGroup, clientSettings.getClientType(), clientSettings.getSubscription().getSubscriptionsList(), false); + break; + } + default: { + future.complete(HeartbeatResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, clientSettings.getClientType().name())) + .build()); + return future; + } + } + future.complete(HeartbeatResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + return future; + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture notifyClientTermination(ProxyContext ctx, + NotifyClientTerminationRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + String clientId = ctx.getClientID(); + LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage()); + Settings clientSettings = grpcClientSettingsManager.removeAndGetClientSettings(ctx); + + switch (clientSettings.getClientType()) { + case PRODUCER: + for (Resource topic : clientSettings.getPublishing().getTopicsList()) { + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(topic); + // user topic name as producer group + GrpcClientChannel channel = this.grpcChannelManager.removeChannel(topicName, clientId); + if (channel != null) { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, MQVersion.Version.V5_0_0.ordinal()); + this.messagingProcessor.unRegisterProducer(ctx, topicName, clientChannelInfo); + } + } + break; + case PUSH_CONSUMER: + case SIMPLE_CONSUMER: + validateConsumerGroup(request.getGroup()); + String consumerGroup = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + GrpcClientChannel channel = this.grpcChannelManager.removeChannel(consumerGroup, clientId); + if (channel != null) { + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, MQVersion.Version.V5_0_0.ordinal()); + this.messagingProcessor.unRegisterConsumer(ctx, consumerGroup, clientChannelInfo); + } + break; + default: + future.complete(NotifyClientTerminationResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.UNRECOGNIZED_CLIENT_TYPE, clientSettings.getClientType().name())) + .build()); + return future; + } + future.complete(NotifyClientTerminationResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public StreamObserver telemetry(ProxyContext ctx, + StreamObserver responseObserver) { + return new StreamObserver() { + @Override + public void onNext(TelemetryCommand request) { + try { + switch (request.getCommandCase()) { + case SETTINGS: { + responseObserver.onNext(processClientSettings(ctx, request, responseObserver)); + break; + } + case THREAD_STACK_TRACE: { + reportThreadStackTrace(ctx, request.getStatus(), request.getThreadStackTrace()); + break; + } + case VERIFY_MESSAGE_RESULT: { + reportVerifyMessageResult(ctx, request.getStatus(), request.getVerifyMessageResult()); + break; + } + } + } catch (Throwable t) { + responseObserver.onNext(convertToTelemetryCommand(t)); + } + } + + @Override + public void onError(Throwable t) { + log.error("telemetry on error", t); + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + + protected TelemetryCommand convertToTelemetryCommand(Throwable t) { + return TelemetryCommand.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(t)).build(); + } + + protected TelemetryCommand processClientSettings(ProxyContext ctx, TelemetryCommand request, + StreamObserver responseObserver) { + String clientId = ctx.getClientID(); + Settings settings = request.getSettings(); + if (settings.hasPublishing()) { + for (Resource topic : settings.getPublishing().getTopicsList()) { + validateTopic(topic); + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(topic); + GrpcClientChannel producerChannel = registerProducer(ctx, topicName); + producerChannel.setClientObserver(responseObserver); + } + } + if (settings.hasSubscription()) { + validateConsumerGroup(settings.getSubscription().getGroup()); + String groupName = GrpcConverter.getInstance().wrapResourceWithNamespace(settings.getSubscription().getGroup()); + GrpcClientChannel consumerChannel = registerConsumer(ctx, groupName, settings.getClientType(), settings.getSubscription().getSubscriptionsList(), true); + consumerChannel.setClientObserver(responseObserver); + } + + grpcClientSettingsManager.updateClientSettings(clientId, request.getSettings()); + settings = grpcClientSettingsManager.getClientSettings(ctx); + return TelemetryCommand.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .setSettings(settings) + .build(); + } + + protected GrpcClientChannel registerProducer(ProxyContext ctx, String topicName) { + String clientId = ctx.getClientID(); + LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage()); + + GrpcClientChannel channel = this.grpcChannelManager.createChannel(ctx, topicName, clientId); + // use topic name as producer group + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, parseClientVersion(ctx.getClientVersion())); + this.messagingProcessor.registerProducer(ctx, topicName, clientChannelInfo); + this.messagingProcessor.addTransactionSubscription(ctx, topicName, topicName); + return channel; + } + + protected GrpcClientChannel registerConsumer(ProxyContext ctx, String consumerGroup, ClientType clientType, List subscriptionEntryList, boolean updateSubscription) { + String clientId = ctx.getClientID(); + LanguageCode languageCode = LanguageCode.valueOf(ctx.getLanguage()); + + GrpcClientChannel channel = this.grpcChannelManager.createChannel(ctx, consumerGroup, clientId); + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(channel, clientId, languageCode, parseClientVersion(ctx.getClientVersion())); + + this.messagingProcessor.registerConsumer( + ctx, + consumerGroup, + clientChannelInfo, + this.buildConsumeType(clientType), + MessageModel.CLUSTERING, + ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, + this.buildSubscriptionDataSet(subscriptionEntryList), + updateSubscription + ); + return channel; + } + + private int parseClientVersion(String clientVersionStr) { + int clientVersion = MQVersion.CURRENT_VERSION; + if (!StringUtils.isEmpty(clientVersionStr)) { + try { + String tmp = StringUtils.upperCase(clientVersionStr); + clientVersion = MQVersion.Version.valueOf(tmp).ordinal(); + } catch (Exception ignored) { + } + } + return clientVersion; + } + + protected void reportThreadStackTrace(ProxyContext ctx, Status status, ThreadStackTrace request) { + String nonce = request.getNonce(); + String threadStack = request.getThreadStackTrace(); + CompletableFuture> responseFuture = this.grpcChannelManager.getAndRemoveResponseFuture(nonce); + if (responseFuture != null) { + try { + if (status.getCode().equals(Code.OK)) { + ConsumerRunningInfo runningInfo = new ConsumerRunningInfo(); + runningInfo.setJstack(threadStack); + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", runningInfo)); + } else if (status.getCode().equals(Code.VERIFY_FIFO_MESSAGE_UNSUPPORTED)) { + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.NO_PERMISSION, "forbidden to verify message", null)); + } else { + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SYSTEM_ERROR, "verify message failed", null)); + } + } catch (Throwable t) { + responseFuture.completeExceptionally(t); + } + } + } + + protected void reportVerifyMessageResult(ProxyContext ctx, Status status, VerifyMessageResult request) { + String nonce = request.getNonce(); + CompletableFuture> responseFuture = this.grpcChannelManager.getAndRemoveResponseFuture(nonce); + if (responseFuture != null) { + try { + ConsumeMessageDirectlyResult result = this.buildConsumeMessageDirectlyResult(status, request); + responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result)); + } catch (Throwable t) { + responseFuture.completeExceptionally(t); + } + } + } + + protected ConsumeMessageDirectlyResult buildConsumeMessageDirectlyResult(Status status, + VerifyMessageResult request) { + ConsumeMessageDirectlyResult consumeMessageDirectlyResult = new ConsumeMessageDirectlyResult(); + switch (status.getCode().getNumber()) { + case Code.OK_VALUE: { + consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_SUCCESS); + break; + } + case Code.FAILED_TO_CONSUME_MESSAGE_VALUE: { + consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_LATER); + break; + } + case Code.MESSAGE_CORRUPTED_VALUE: { + consumeMessageDirectlyResult.setConsumeResult(CMResult.CR_RETURN_NULL); + break; + } + } + consumeMessageDirectlyResult.setRemark("from gRPC client"); + return consumeMessageDirectlyResult; + } + + protected ConsumeType buildConsumeType(ClientType clientType) { + switch (clientType) { + case SIMPLE_CONSUMER: + return ConsumeType.CONSUME_ACTIVELY; + case PUSH_CONSUMER: + return ConsumeType.CONSUME_PASSIVELY; + default: + throw new IllegalArgumentException("Client type is not consumer, type: " + clientType); + } + } + + protected Set buildSubscriptionDataSet(List subscriptionEntryList) { + Set subscriptionDataSet = new HashSet<>(); + for (SubscriptionEntry sub : subscriptionEntryList) { + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(sub.getTopic()); + FilterExpression filterExpression = sub.getExpression(); + subscriptionDataSet.add(buildSubscriptionData(topicName, filterExpression)); + } + return subscriptionDataSet; + } + + protected SubscriptionData buildSubscriptionData(String topicName, FilterExpression filterExpression) { + String expression = filterExpression.getExpression(); + String expressionType = GrpcConverter.getInstance().buildExpressionType(filterExpression.getType()); + try { + return FilterAPI.build(topicName, expression, expressionType); + } catch (Exception e) { + throw new GrpcProxyException(Code.ILLEGAL_FILTER_EXPRESSION, "expression format is not correct", e); + } + } + + protected class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener { + + @Override + public void handle(ConsumerGroupEvent event, String group, Object... args) { + if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { + if (args == null || args.length < 1) { + return; + } + if (args[0] instanceof ClientChannelInfo) { + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; + grpcChannelManager.removeChannel(group, clientChannelInfo.getClientId()); + grpcClientSettingsManager.removeClientSettings(clientChannelInfo.getClientId()); + } + } + } + + @Override + public void shutdown() { + + } + } + + protected class ProducerChangeListenerImpl implements ProducerChangeListener { + + @Override + public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) { + if (event == ProducerGroupEvent.CLIENT_UNREGISTER) { + grpcChannelManager.removeChannel(group, clientChannelInfo.getClientId()); + grpcClientSettingsManager.removeClientSettings(clientChannelInfo.getClientId()); + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java new file mode 100644 index 0000000000..548bd5efd2 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManager.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.Address; +import apache.rocketmq.v2.AddressScheme; +import apache.rocketmq.v2.CustomizedBackoff; +import apache.rocketmq.v2.Endpoints; +import apache.rocketmq.v2.ExponentialBackoff; +import apache.rocketmq.v2.Metric; +import apache.rocketmq.v2.Publishing; +import apache.rocketmq.v2.RetryPolicy; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import com.google.protobuf.Duration; +import com.google.protobuf.util.Durations; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.rocketmq.common.subscription.CustomizedRetryPolicy; +import org.apache.rocketmq.common.subscription.ExponentialRetryPolicy; +import org.apache.rocketmq.common.subscription.GroupRetryPolicy; +import org.apache.rocketmq.common.subscription.GroupRetryPolicyType; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.MetricCollectorMode; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; + +public class GrpcClientSettingsManager { + + protected static final Map CLIENT_SETTINGS_MAP = new ConcurrentHashMap<>(); + + private final MessagingProcessor messagingProcessor; + + public GrpcClientSettingsManager(MessagingProcessor messagingProcessor) { + this.messagingProcessor = messagingProcessor; + } + + public Settings getClientSettings(ProxyContext ctx) { + String clientId = ctx.getClientID(); + Settings settings = CLIENT_SETTINGS_MAP.get(clientId); + if (settings == null) { + return null; + } + if (settings.hasSubscription()) { + settings = mergeSubscriptionData(ctx, settings, + GrpcConverter.getInstance().wrapResourceWithNamespace(settings.getSubscription().getGroup())); + } + return mergeMetric(settings); + } + + private Settings mergeSubscriptionData(ProxyContext ctx, Settings settings, String consumerGroup) { + SubscriptionGroupConfig config = this.messagingProcessor.getSubscriptionGroupConfig(ctx, consumerGroup); + if (config == null) { + return settings; + } + + return mergeSubscriptionData(settings, config); + } + + private Settings mergeMetric(Settings settings) { + // Construct metric according to the proxy config + final ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + final MetricCollectorMode metricCollectorMode = + MetricCollectorMode.getEnumByOrdinal(proxyConfig.getMetricCollectorMode()); + final String metricCollectorAddress = proxyConfig.getMetricCollectorAddress(); + final Metric.Builder metricBuilder = Metric.newBuilder(); + switch (metricCollectorMode) { + case ON: + final String[] split = metricCollectorAddress.split(":"); + final String host = split[0]; + final int port = Integer.parseInt(split[1]); + Address address = Address.newBuilder().setHost(host).setPort(port).build(); + final Endpoints endpoints = Endpoints.newBuilder().setScheme(AddressScheme.IPv4) + .addAddresses(address).build(); + metricBuilder.setOn(true).setEndpoints(endpoints); + break; + case PROXY: + metricBuilder.setOn(true).setEndpoints(settings.getAccessPoint()); + break; + case OFF: + default: + metricBuilder.setOn(false); + break; + } + Metric metric = metricBuilder.build(); + return settings.toBuilder().setMetric(metric).build(); + } + + protected static Settings mergeSubscriptionData(Settings settings, SubscriptionGroupConfig config) { + Settings.Builder resultSettingsBuilder = settings.toBuilder(); + + resultSettingsBuilder.getSubscriptionBuilder().setFifo(config.isConsumeMessageOrderly()); + + resultSettingsBuilder.getBackoffPolicyBuilder().setMaxAttempts(config.getRetryMaxTimes() + 1); + + GroupRetryPolicy groupRetryPolicy = config.getGroupRetryPolicy(); + if (groupRetryPolicy.getType().equals(GroupRetryPolicyType.EXPONENTIAL)) { + ExponentialRetryPolicy exponentialRetryPolicy = groupRetryPolicy.getExponentialRetryPolicy(); + if (exponentialRetryPolicy == null) { + exponentialRetryPolicy = new ExponentialRetryPolicy(); + } + resultSettingsBuilder.getBackoffPolicyBuilder().setExponentialBackoff(convertToExponentialBackoff(exponentialRetryPolicy)); + } else { + CustomizedRetryPolicy customizedRetryPolicy = groupRetryPolicy.getCustomizedRetryPolicy(); + if (customizedRetryPolicy == null) { + customizedRetryPolicy = new CustomizedRetryPolicy(); + } + resultSettingsBuilder.getBackoffPolicyBuilder().setCustomizedBackoff(convertToCustomizedRetryPolicy(customizedRetryPolicy)); + } + + return resultSettingsBuilder.build(); + } + + protected static ExponentialBackoff convertToExponentialBackoff(ExponentialRetryPolicy retryPolicy) { + return ExponentialBackoff.newBuilder() + .setInitial(Durations.fromMillis(retryPolicy.getInitial())) + .setMax(Durations.fromMillis(retryPolicy.getMax())) + .setMultiplier(retryPolicy.getMultiplier()) + .build(); + } + + protected static CustomizedBackoff convertToCustomizedRetryPolicy(CustomizedRetryPolicy retryPolicy) { + List durationList = Arrays.stream(retryPolicy.getNext()) + .mapToObj(Durations::fromMillis).collect(Collectors.toList()); + return CustomizedBackoff.newBuilder() + .addAllNext(durationList) + .build(); + } + + public void updateClientSettings(String clientId, Settings settings) { + if (settings.hasPublishing()) { + settings = createDefaultProducerSettingsBuilder().mergeFrom(settings).build(); + } else if (settings.hasSubscription()) { + settings = createDefaultConsumerSettingsBuilder().mergeFrom(settings).build(); + } + CLIENT_SETTINGS_MAP.put(clientId, settings); + } + + protected Settings.Builder createDefaultProducerSettingsBuilder() { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + return Settings.newBuilder() + .setBackoffPolicy(RetryPolicy.newBuilder() + .setMaxAttempts(config.getGrpcClientProducerMaxAttempts()) + .setExponentialBackoff(ExponentialBackoff.newBuilder() + .setInitial(Durations.fromMillis(config.getGrpcClientProducerBackoffInitialMillis())) + .setMax(Durations.fromMillis(config.getGrpcClientProducerBackoffMaxMillis())) + .setMultiplier(config.getGrpcClientProducerBackoffMultiplier()) + .build()) + .build()) + .setPublishing(Publishing.newBuilder() + .setValidateMessageType(config.isEnableTopicMessageTypeCheck()) + .setMaxBodySize(config.getMaxMessageSize()) + .build()); + } + + protected Settings.Builder createDefaultConsumerSettingsBuilder() { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + return mergeSubscriptionData(Settings.newBuilder() + .setSubscription(Subscription.newBuilder() + .setReceiveBatchSize(config.getGrpcClientConsumerLongPollingBatchSize()) + .setLongPollingTimeout(Durations.fromMillis(config.getGrpcClientConsumerLongPollingTimeoutMillis())) + .build()) + .build(), new SubscriptionGroupConfig()) + .toBuilder(); + } + + public void removeClientSettings(String clientId) { + CLIENT_SETTINGS_MAP.remove(clientId); + } + + public Settings removeAndGetClientSettings(ProxyContext ctx) { + String clientId = ctx.getClientID(); + Settings settings = CLIENT_SETTINGS_MAP.remove(clientId); + if (settings == null) { + return null; + } + settings = mergeSubscriptionData(ctx, settings, + GrpcConverter.getInstance().wrapResourceWithNamespace(settings.getSubscription().getGroup())); + if (settings == null) { + return null; + } + return mergeMetric(settings); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java new file mode 100644 index 0000000000..cc5a60ca6d --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverter.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.Broker; +import apache.rocketmq.v2.Digest; +import apache.rocketmq.v2.DigestType; +import apache.rocketmq.v2.Encoding; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.Message; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.SystemProperties; +import com.google.protobuf.ByteString; +import com.google.protobuf.util.Timestamps; +import java.net.SocketAddress; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.utils.BinaryUtil; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.common.RemotingUtil; + +public class GrpcConverter { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected static final Object INSTANCE_CREATE_LOCK = new Object(); + protected static volatile GrpcConverter instance; + + public static GrpcConverter getInstance() { + if (instance == null) { + synchronized (INSTANCE_CREATE_LOCK) { + if (instance == null) { + instance = new GrpcConverter(); + } + } + } + return instance; + } + + public String wrapResourceWithNamespace(Resource resource) { + return NamespaceUtil.wrapNamespace(resource.getResourceNamespace(), resource.getName()); + } + + public MessageQueue buildMessageQueue(MessageExt messageExt, String brokerName) { + Broker broker = Broker.getDefaultInstance(); + if (!StringUtils.isEmpty(brokerName)) { + broker = Broker.newBuilder() + .setName(brokerName) + .setId(0) + .build(); + } + return MessageQueue.newBuilder() + .setId(messageExt.getQueueId()) + .setTopic(Resource.newBuilder() + .setName(NamespaceUtil.withoutNamespace(messageExt.getTopic())) + .setResourceNamespace(NamespaceUtil.getNamespaceFromResource(messageExt.getTopic())) + .build()) + .setBroker(broker) + .build(); + } + + public String buildExpressionType(FilterType filterType) { + switch (filterType) { + case SQL: + return ExpressionType.SQL92; + case TAG: + default: + return ExpressionType.TAG; + } + } + + public Message buildMessage(MessageExt messageExt) { + Map userProperties = buildUserAttributes(messageExt); + SystemProperties systemProperties = buildSystemProperties(messageExt); + Resource topic = buildResource(messageExt.getTopic()); + + return Message.newBuilder() + .setTopic(topic) + .putAllUserProperties(userProperties) + .setSystemProperties(systemProperties) + .setBody(ByteString.copyFrom(messageExt.getBody())) + .build(); + } + + protected Map buildUserAttributes(MessageExt messageExt) { + Map userAttributes = new HashMap<>(); + Map properties = messageExt.getProperties(); + + for (Map.Entry property : properties.entrySet()) { + if (!MessageConst.STRING_HASH_SET.contains(property.getKey())) { + userAttributes.put(property.getKey(), property.getValue()); + } + } + + return userAttributes; + } + + protected SystemProperties buildSystemProperties(MessageExt messageExt) { + SystemProperties.Builder systemPropertiesBuilder = SystemProperties.newBuilder(); + + // tag + String tag = messageExt.getUserProperty(MessageConst.PROPERTY_TAGS); + if (tag != null) { + systemPropertiesBuilder.setTag(tag); + } + + // keys + String keys = messageExt.getKeys(); + if (keys != null) { + String[] keysArray = keys.split(MessageConst.KEY_SEPARATOR); + systemPropertiesBuilder.addAllKeys(Arrays.asList(keysArray)); + } + + // message_id + String uniqKey = messageExt.getProperty(MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX); + if (uniqKey != null) { + systemPropertiesBuilder.setMessageId(uniqKey); + } + + // body_digest & body_encoding + String md5Result = BinaryUtil.generateMd5(messageExt.getBody()); + Digest digest = Digest.newBuilder() + .setType(DigestType.MD5) + .setChecksum(md5Result) + .build(); + systemPropertiesBuilder.setBodyDigest(digest); + + if ((messageExt.getSysFlag() & MessageSysFlag.COMPRESSED_FLAG) == MessageSysFlag.COMPRESSED_FLAG) { + systemPropertiesBuilder.setBodyEncoding(Encoding.GZIP); + } else { + systemPropertiesBuilder.setBodyEncoding(Encoding.IDENTITY); + } + + // message_type + String isTrans = messageExt.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + String isTransValue = "true"; + if (isTransValue.equals(isTrans)) { + systemPropertiesBuilder.setMessageType(MessageType.TRANSACTION); + } else if (messageExt.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null + || messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null + || messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) { + systemPropertiesBuilder.setMessageType(MessageType.DELAY); + } else if (messageExt.getProperty(MessageConst.PROPERTY_SHARDING_KEY) != null) { + systemPropertiesBuilder.setMessageType(MessageType.FIFO); + } else { + systemPropertiesBuilder.setMessageType(MessageType.NORMAL); + } + + // born_timestamp (millis) + long bornTimestamp = messageExt.getBornTimestamp(); + systemPropertiesBuilder.setBornTimestamp(Timestamps.fromMillis(bornTimestamp)); + + // born_host + String bornHostString = messageExt.getProperty(MessageConst.PROPERTY_BORN_HOST); + if (StringUtils.isBlank(bornHostString)) { + bornHostString = messageExt.getBornHostString(); + } + if (StringUtils.isNotBlank(bornHostString)) { + systemPropertiesBuilder.setBornHost(bornHostString); + } + + // store_timestamp (millis) + long storeTimestamp = messageExt.getStoreTimestamp(); + systemPropertiesBuilder.setStoreTimestamp(Timestamps.fromMillis(storeTimestamp)); + + // store_host + SocketAddress storeHost = messageExt.getStoreHost(); + if (storeHost != null) { + systemPropertiesBuilder.setStoreHost(RemotingUtil.socketAddress2String(storeHost)); + } + + // delivery_timestamp + String deliverMsString; + long deliverMs; + if (messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) { + long delayMs = TimeUnit.SECONDS.toMillis(Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC))); + deliverMs = System.currentTimeMillis() + delayMs; + systemPropertiesBuilder.setDeliveryTimestamp(Timestamps.fromMillis(deliverMs)); + } else { + deliverMsString = messageExt.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS); + if (deliverMsString != null) { + deliverMs = Long.parseLong(deliverMsString); + systemPropertiesBuilder.setDeliveryTimestamp(Timestamps.fromMillis(deliverMs)); + } + } + + // sharding key + String shardingKey = messageExt.getProperty(MessageConst.PROPERTY_SHARDING_KEY); + if (shardingKey != null) { + systemPropertiesBuilder.setMessageGroup(shardingKey); + } + + // receipt_handle && invisible_period + String handle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK); + if (handle != null) { + systemPropertiesBuilder.setReceiptHandle(handle); + } + + // partition_id + systemPropertiesBuilder.setQueueId(messageExt.getQueueId()); + + // partition_offset + systemPropertiesBuilder.setQueueOffset(messageExt.getQueueOffset()); + + // delivery_attempt + systemPropertiesBuilder.setDeliveryAttempt(messageExt.getReconsumeTimes() + 1); + + // trace context + String traceContext = messageExt.getProperty(MessageConst.PROPERTY_TRACE_CONTEXT); + if (traceContext != null) { + systemPropertiesBuilder.setTraceContext(traceContext); + } + + return systemPropertiesBuilder.build(); + } + + public Resource buildResource(String resourceNameWithNamespace) { + return Resource.newBuilder() + .setResourceNamespace(NamespaceUtil.getNamespaceFromResource(resourceNameWithNamespace)) + .setName(NamespaceUtil.withoutNamespace(resourceNameWithNamespace)) + .build(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcProxyException.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcProxyException.java new file mode 100644 index 0000000000..74e499b4d7 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcProxyException.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.Code; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + +public class GrpcProxyException extends RuntimeException { + + private ProxyException proxyException; + private Code code; + + protected static final Map CODE_MAPPING = new ConcurrentHashMap<>(); + + static { + CODE_MAPPING.put(ProxyExceptionCode.INVALID_BROKER_NAME, Code.BAD_REQUEST); + CODE_MAPPING.put(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, Code.INVALID_RECEIPT_HANDLE); + CODE_MAPPING.put(ProxyExceptionCode.FORBIDDEN, Code.FORBIDDEN); + CODE_MAPPING.put(ProxyExceptionCode.INTERNAL_SERVER_ERROR, Code.INTERNAL_SERVER_ERROR); + CODE_MAPPING.put(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, Code.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE); + } + + public GrpcProxyException(Code code, String message) { + super(message); + this.code = code; + } + + public GrpcProxyException(Code code, String message, Throwable t) { + super(message, t); + this.code = code; + } + + public GrpcProxyException(ProxyException proxyException) { + super(proxyException); + this.proxyException = proxyException; + } + + public Code getCode() { + if (this.code != null) { + return this.code; + } + if (this.proxyException != null) { + return CODE_MAPPING.getOrDefault(this.proxyException.getCode(), Code.INTERNAL_SERVER_ERROR); + } + return Code.INTERNAL_SERVER_ERROR; + } + + public ProxyException getProxyException() { + return proxyException; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcValidator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcValidator.java new file mode 100644 index 0000000000..0ada96b864 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcValidator.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Resource; +import com.google.common.base.CharMatcher; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.Validators; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.config.ConfigurationManager; + +public class GrpcValidator { + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected static final Object INSTANCE_CREATE_LOCK = new Object(); + protected static volatile GrpcValidator instance; + + public static GrpcValidator getInstance() { + if (instance == null) { + synchronized (INSTANCE_CREATE_LOCK) { + if (instance == null) { + instance = new GrpcValidator(); + } + } + } + return instance; + } + + public void validateTopic(Resource topic) { + validateTopic(GrpcConverter.getInstance().wrapResourceWithNamespace(topic)); + } + + public void validateTopic(String topicName) { + if (StringUtils.isBlank(topicName)) { + throw new GrpcProxyException(Code.ILLEGAL_TOPIC, "topic name cannot be empty"); + } + if (TopicValidator.isSystemTopic(topicName)) { + throw new GrpcProxyException(Code.ILLEGAL_TOPIC, "cannot access system topic"); + } + try { + Validators.checkTopic(topicName); + } catch (MQClientException mqClientException) { + throw new GrpcProxyException(Code.ILLEGAL_TOPIC, mqClientException.getErrorMessage()); + } + } + + public void validateConsumerGroup(Resource consumerGroup) { + validateConsumerGroup(GrpcConverter.getInstance().wrapResourceWithNamespace(consumerGroup)); + } + + public void validateConsumerGroup(String consumerGroupName) { + if (StringUtils.isBlank(consumerGroupName)) { + throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, "consumer group cannot be empty"); + } + if (MixAll.isSysConsumerGroup(consumerGroupName)) { + throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, "cannot use system consumer group"); + } + try { + Validators.checkGroup(consumerGroupName); + } catch (MQClientException mqClientException) { + throw new GrpcProxyException(Code.ILLEGAL_CONSUMER_GROUP, mqClientException.getErrorMessage()); + } + } + + public void validateTopicAndConsumerGroup(Resource topic, Resource consumerGroup) { + validateTopic(topic); + validateConsumerGroup(consumerGroup); + } + + public void validateInvisibleTime(long invisibleTime) { + validateInvisibleTime(invisibleTime, 0); + } + + public void validateInvisibleTime(long invisibleTime, long minInvisibleTime) { + if (invisibleTime < minInvisibleTime) { + throw new GrpcProxyException(Code.ILLEGAL_INVISIBLE_TIME, "the invisibleTime is too small. min is " + minInvisibleTime); + } + long maxInvisibleTime = ConfigurationManager.getProxyConfig().getMaxInvisibleTimeMills(); + if (maxInvisibleTime <= 0) { + return; + } + if (invisibleTime > maxInvisibleTime) { + throw new GrpcProxyException(Code.ILLEGAL_INVISIBLE_TIME, "the invisibleTime is too large. max is " + maxInvisibleTime); + } + } + + public void validateTag(String tag) { + if (StringUtils.isNotEmpty(tag)) { + if (StringUtils.isBlank(tag)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, "tag cannot be the char sequence of whitespace"); + } + if (tag.contains("|")) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, "tag cannot contain '|'"); + } + if (containControlCharacter(tag)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_TAG, "tag cannot contain control character"); + } + } + } + + public boolean containControlCharacter(String data) { + for (int i = 0; i < data.length(); i++) { + if (CharMatcher.javaIsoControl().matches(data.charAt(i))) { + return true; + } + } + return false; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java new file mode 100644 index 0000000000..08fa124be7 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Status; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; +import org.apache.rocketmq.proxy.service.route.TopicRouteHelper; +import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; + +public class ResponseBuilder { + + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected static final Map RESPONSE_CODE_MAPPING = new ConcurrentHashMap<>(); + + protected static final Object INSTANCE_CREATE_LOCK = new Object(); + protected static volatile ResponseBuilder instance; + + static { + RESPONSE_CODE_MAPPING.put(ResponseCode.SUCCESS, Code.OK); + RESPONSE_CODE_MAPPING.put(ResponseCode.SYSTEM_BUSY, Code.TOO_MANY_REQUESTS); + RESPONSE_CODE_MAPPING.put(ResponseCode.REQUEST_CODE_NOT_SUPPORTED, Code.NOT_IMPLEMENTED); + RESPONSE_CODE_MAPPING.put(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST, Code.CONSUMER_GROUP_NOT_FOUND); + RESPONSE_CODE_MAPPING.put(ClientErrorCode.ACCESS_BROKER_TIMEOUT, Code.PROXY_TIMEOUT); + } + + public static ResponseBuilder getInstance() { + if (instance == null) { + synchronized (INSTANCE_CREATE_LOCK) { + if (instance == null) { + instance = new ResponseBuilder(); + } + } + } + return instance; + } + + public Status buildStatus(Throwable t) { + t = ExceptionUtils.getRealException(t); + + if (t instanceof ProxyException) { + t = new GrpcProxyException((ProxyException) t); + } + if (t instanceof GrpcProxyException) { + GrpcProxyException grpcProxyException = (GrpcProxyException) t; + return buildStatus(grpcProxyException.getCode(), grpcProxyException.getMessage()); + } + if (TopicRouteHelper.isTopicNotExistError(t)) { + return buildStatus(Code.TOPIC_NOT_FOUND, t.getMessage()); + } + if (t instanceof MQBrokerException) { + MQBrokerException mqBrokerException = (MQBrokerException) t; + return buildStatus(buildCode(mqBrokerException.getResponseCode()), mqBrokerException.getErrorMessage()); + } + if (t instanceof MQClientException) { + MQClientException mqClientException = (MQClientException) t; + return buildStatus(buildCode(mqClientException.getResponseCode()), mqClientException.getErrorMessage()); + } + if (t instanceof RemotingTimeoutException) { + return buildStatus(Code.PROXY_TIMEOUT, t.getMessage()); + } + + log.error("internal server error", t); + return buildStatus(Code.INTERNAL_SERVER_ERROR, ExceptionUtils.getErrorDetailMessage(t)); + } + + public Status buildStatus(Code code, String message) { + return Status.newBuilder() + .setCode(code) + .setMessage(message) + .build(); + } + + public Status buildStatus(int remotingResponseCode, String remark) { + String message = remark; + if (message == null) { + message = String.valueOf(remotingResponseCode); + } + return Status.newBuilder() + .setCode(buildCode(remotingResponseCode)) + .setMessage(message) + .build(); + } + + public Code buildCode(int remotingResponseCode) { + return RESPONSE_CODE_MAPPING.getOrDefault(remotingResponseCode, Code.INTERNAL_SERVER_ERROR); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseWriter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseWriter.java new file mode 100644 index 0000000000..43ddf99992 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseWriter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +public class ResponseWriter { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected static final Object INSTANCE_CREATE_LOCK = new Object(); + protected static volatile ResponseWriter instance; + + public static ResponseWriter getInstance() { + if (instance == null) { + synchronized (INSTANCE_CREATE_LOCK) { + if (instance == null) { + instance = new ResponseWriter(); + } + } + } + return instance; + } + + public void write(StreamObserver observer, final T response) { + if (writeResponse(observer, response)) { + observer.onCompleted(); + } + } + + public boolean writeResponse(StreamObserver observer, final T response) { + if (null == response) { + return false; + } + log.debug("start to write response. response: {}", response); + if (isCancelled(observer)) { + log.warn("client has cancelled the request. response to write: {}", response); + return false; + } + try { + observer.onNext(response); + } catch (StatusRuntimeException statusRuntimeException) { + if (Status.CANCELLED.equals(statusRuntimeException.getStatus())) { + log.warn("client has cancelled the request. response to write: {}", response); + return false; + } + throw statusRuntimeException; + } + return true; + } + + public boolean isCancelled(StreamObserver observer) { + if (observer instanceof ServerCallStreamObserver) { + final ServerCallStreamObserver serverCallStreamObserver = (ServerCallStreamObserver) observer; + return serverCallStreamObserver.isCancelled(); + } + return false; + } +} + diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java new file mode 100644 index 0000000000..5f4c85af7c --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.AckMessageEntry; +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.AckMessageResultEntry; +import apache.rocketmq.v2.Code; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.proxy.common.MessageReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor; + +public class AckMessageActivity extends AbstractMessingActivity { + protected ReceiptHandleProcessor receiptHandleProcessor; + + public AckMessageActivity(MessagingProcessor messagingProcessor, ReceiptHandleProcessor receiptHandleProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, + GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.receiptHandleProcessor = receiptHandleProcessor; + } + + public CompletableFuture ackMessage(ProxyContext ctx, AckMessageRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); + + CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; + for (int i = 0; i < request.getEntriesCount(); i++) { + futures[i] = processAckMessage(ctx, request, request.getEntries(i)); + } + CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { + if (throwable != null) { + future.completeExceptionally(throwable); + return; + } + + Set responseCodes = new HashSet<>(); + List entryList = new ArrayList<>(); + for (CompletableFuture entryFuture : futures) { + AckMessageResultEntry entryResult = entryFuture.join(); + responseCodes.add(entryResult.getStatus().getCode()); + entryList.add(entryResult); + } + AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() + .addAllEntries(entryList); + if (responseCodes.size() > 1) { + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); + } else if (responseCodes.size() == 1) { + Code code = responseCodes.stream().findAny().get(); + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); + } else { + responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); + } + future.complete(responseBuilder.build()); + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, + AckMessageEntry ackMessageEntry) { + CompletableFuture future = new CompletableFuture<>(); + + try { + String handleString = ackMessageEntry.getReceiptHandle(); + + String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + MessageReceiptHandle messageReceiptHandle = receiptHandleProcessor.removeReceiptHandle(ctx.getClientID(), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); + if (messageReceiptHandle != null) { + handleString = messageReceiptHandle.getReceiptHandle(); + } + CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( + ctx, + ReceiptHandle.decode(handleString), + ackMessageEntry.getMessageId(), + group, + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); + ackResultFuture.thenAccept(result -> { + future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); + }).exceptionally(t -> { + future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, t)); + return null; + }); + } catch (Throwable t) { + future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, t)); + } + return future; + } + + protected AckMessageResultEntry convertToAckMessageResultEntry(ProxyContext ctx, AckMessageEntry ackMessageEntry, Throwable throwable) { + return AckMessageResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(throwable)) + .setMessageId(ackMessageEntry.getMessageId()) + .setReceiptHandle(ackMessageEntry.getReceiptHandle()) + .build(); + } + + protected AckMessageResultEntry convertToAckMessageResultEntry(ProxyContext ctx, AckMessageEntry ackMessageEntry, + AckResult ackResult) { + if (AckStatus.OK.equals(ackResult.getStatus())) { + return AckMessageResultEntry.newBuilder() + .setMessageId(ackMessageEntry.getMessageId()) + .setReceiptHandle(ackMessageEntry.getReceiptHandle()) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build(); + } + return AckMessageResultEntry.newBuilder() + .setMessageId(ackMessageEntry.getMessageId()) + .setReceiptHandle(ackMessageEntry.getReceiptHandle()) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) + .build(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivity.java new file mode 100644 index 0000000000..ff63127bc6 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivity.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.Code; +import com.google.protobuf.util.Durations; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor; + +public class ChangeInvisibleDurationActivity extends AbstractMessingActivity { + protected ReceiptHandleProcessor receiptHandleProcessor; + + public ChangeInvisibleDurationActivity(MessagingProcessor messagingProcessor, + ReceiptHandleProcessor receiptHandleProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.receiptHandleProcessor = receiptHandleProcessor; + } + + public CompletableFuture changeInvisibleDuration(ProxyContext ctx, + ChangeInvisibleDurationRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); + validateInvisibleTime(Durations.toMillis(request.getInvisibleDuration())); + + ReceiptHandle receiptHandle = ReceiptHandle.decode(request.getReceiptHandle()); + + String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + return this.messagingProcessor.changeInvisibleTime( + ctx, + receiptHandle, + request.getMessageId(), + group, + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()), + Durations.toMillis(request.getInvisibleDuration()) + ).thenApply(ackResult -> { + if (AckStatus.OK.equals(ackResult.getStatus())) { + receiptHandleProcessor.removeReceiptHandle(ctx.getClientID(), group, request.getMessageId(), receiptHandle.getReceiptHandle()); + } + return convertToChangeInvisibleDurationResponse(ctx, request, ackResult); + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected ChangeInvisibleDurationResponse convertToChangeInvisibleDurationResponse(ProxyContext ctx, + ChangeInvisibleDurationRequest request, AckResult ackResult) { + if (AckStatus.OK.equals(ackResult.getStatus())) { + return ChangeInvisibleDurationResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .setReceiptHandle(ackResult.getExtraInfo()) + .build(); + } + return ChangeInvisibleDurationResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "changeInvisibleDuration failed: status is abnormal")) + .build(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/PopMessageResultFilterImpl.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/PopMessageResultFilterImpl.java new file mode 100644 index 0000000000..3f9f2b2174 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/PopMessageResultFilterImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.utils.FilterUtils; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.processor.PopMessageResultFilter; + +public class PopMessageResultFilterImpl implements PopMessageResultFilter { + + private final GrpcClientSettingsManager grpcClientSettingsManager; + + public PopMessageResultFilterImpl(GrpcClientSettingsManager manager) { + grpcClientSettingsManager = manager; + } + + @Override + public FilterResult filterMessage(ProxyContext ctx, String consumerGroup, SubscriptionData subscriptionData, + MessageExt messageExt) { + int maxAttempts = grpcClientSettingsManager.getClientSettings(ctx).getBackoffPolicy().getMaxAttempts(); + if (!FilterUtils.isTagMatched(subscriptionData.getTagsSet(), messageExt.getTags())) { + return FilterResult.NO_MATCH; + } + if (messageExt.getReconsumeTimes() >= maxAttempts) { + return FilterResult.TO_DLQ; + } + return FilterResult.MATCH; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivity.java new file mode 100644 index 0000000000..b69db78b11 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivity.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import com.google.protobuf.util.Durations; +import io.grpc.stub.StreamObserver; +import java.time.Duration; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.constant.ConsumeInitMode; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.proxy.common.MessageReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.QueueSelector; +import org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueSelector; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; + +public class ReceiveMessageActivity extends AbstractMessingActivity { + protected ReceiptHandleProcessor receiptHandleProcessor; + + public ReceiveMessageActivity(MessagingProcessor messagingProcessor, ReceiptHandleProcessor receiptHandleProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + this.receiptHandleProcessor = receiptHandleProcessor; + } + + public void receiveMessage(ProxyContext ctx, ReceiveMessageRequest request, + StreamObserver responseObserver) { + ReceiveMessageResponseStreamWriter writer = createWriter(ctx, responseObserver); + + try { + Settings settings = this.grpcClientSettingsManager.getClientSettings(ctx); + Subscription subscription = settings.getSubscription(); + boolean fifo = subscription.getFifo(); + ProxyConfig config = ConfigurationManager.getProxyConfig(); + + Long timeRemaining = ctx.getRemainingMs(); + if (timeRemaining == null) { + timeRemaining = Duration.ofSeconds(20).toMillis(); + } + long pollTime = timeRemaining - config.getLongPollingReserveTimeInMillis(); + if (pollTime <= 0) { + writer.writeAndComplete(ctx, Code.BAD_REQUEST, "time remaining is too small"); + return; + } + if (pollTime > config.getGrpcClientConsumerLongPollingTimeoutMillis()) { + pollTime = config.getGrpcClientConsumerLongPollingTimeoutMillis(); + } + + validateTopicAndConsumerGroup(request.getMessageQueue().getTopic(), request.getGroup()); + String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getMessageQueue().getTopic()); + String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); + + long actualInvisibleTime = Durations.toMillis(request.getInvisibleDuration()); + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + if (proxyConfig.isEnableProxyAutoRenew() && request.getAutoRenew()) { + actualInvisibleTime = proxyConfig.getRenewSliceTimeMillis(); + } else { + validateInvisibleTime(actualInvisibleTime, + ConfigurationManager.getProxyConfig().getMinInvisibleTimeMillsForRecv()); + } + + FilterExpression filterExpression = request.getFilterExpression(); + SubscriptionData subscriptionData; + try { + subscriptionData = FilterAPI.build(topic, filterExpression.getExpression(), + GrpcConverter.getInstance().buildExpressionType(filterExpression.getType())); + } catch (Exception e) { + writer.writeAndComplete(ctx, Code.ILLEGAL_FILTER_EXPRESSION, e.getMessage()); + return; + } + + this.messagingProcessor.popMessage( + ctx, + new ReceiveMessageQueueSelector( + request.getMessageQueue().getBroker().getName() + ), + group, + topic, + request.getBatchSize(), + actualInvisibleTime, + pollTime, + ConsumeInitMode.MAX, + subscriptionData, + fifo, + new PopMessageResultFilterImpl(grpcClientSettingsManager), + timeRemaining + ).thenAccept(popResult -> { + if (proxyConfig.isEnableProxyAutoRenew() && request.getAutoRenew()) { + if (PopStatus.FOUND.equals(popResult.getPopStatus())) { + List messageExtList = popResult.getMsgFoundList(); + for (MessageExt messageExt : messageExtList) { + String receiptHandle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK); + if (receiptHandle != null) { + MessageReceiptHandle messageReceiptHandle = + new MessageReceiptHandle(group, topic, messageExt.getQueueId(), receiptHandle, messageExt.getMsgId(), + messageExt.getQueueOffset(), messageExt.getReconsumeTimes(), proxyConfig.getRenewMaxTimeMillis()); + receiptHandleProcessor.addReceiptHandle(ctx.getClientID(), group, messageExt.getMsgId(), receiptHandle, messageReceiptHandle); + } + } + } + } + writer.writeAndComplete(ctx, request, popResult); + }) + .exceptionally(t -> { + writer.writeAndComplete(ctx, request, t); + return null; + }); + } catch (Throwable t) { + writer.writeAndComplete(ctx, request, t); + } + } + + protected ReceiveMessageResponseStreamWriter createWriter(ProxyContext ctx, + StreamObserver responseObserver) { + return new ReceiveMessageResponseStreamWriter( + this.messagingProcessor, + responseObserver + ); + } + + protected static class ReceiveMessageQueueSelector implements QueueSelector { + + private final String brokerName; + + public ReceiveMessageQueueSelector(String brokerName) { + this.brokerName = brokerName; + } + + @Override + public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView) { + try { + AddressableMessageQueue addressableMessageQueue = null; + MessageQueueSelector messageQueueSelector = messageQueueView.getReadSelector(); + + if (StringUtils.isNotBlank(brokerName)) { + addressableMessageQueue = messageQueueSelector.getQueueByBrokerName(brokerName); + } + + if (addressableMessageQueue == null) { + addressableMessageQueue = messageQueueSelector.selectOne(true); + } + return addressableMessageQueue; + } catch (Throwable t) { + return null; + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriter.java new file mode 100644 index 0000000000..7faf1b41c6 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriter.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Message; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import com.google.protobuf.util.Timestamps; +import io.grpc.stub.StreamObserver; +import java.time.Duration; +import java.util.Iterator; +import java.util.List; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseWriter; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; + +public class ReceiveMessageResponseStreamWriter { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected static final long NACK_INVISIBLE_TIME = Duration.ofSeconds(1).toMillis(); + + protected final MessagingProcessor messagingProcessor; + protected final StreamObserver streamObserver; + + public ReceiveMessageResponseStreamWriter( + MessagingProcessor messagingProcessor, + StreamObserver observer) { + this.messagingProcessor = messagingProcessor; + this.streamObserver = observer; + } + + public void writeAndComplete(ProxyContext ctx, ReceiveMessageRequest request, PopResult popResult) { + PopStatus status = popResult.getPopStatus(); + List messageFoundList = popResult.getMsgFoundList(); + try { + switch (status) { + case FOUND: + if (messageFoundList.isEmpty()) { + streamObserver.onNext(ReceiveMessageResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MESSAGE_NOT_FOUND, "no match message")) + .build()); + } else { + streamObserver.onNext(ReceiveMessageResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + Iterator messageIterator = messageFoundList.iterator(); + while (messageIterator.hasNext()) { + MessageExt curMessageExt = messageIterator.next(); + Message curMessage = convertToMessage(curMessageExt); + try { + streamObserver.onNext(ReceiveMessageResponse.newBuilder() + .setMessage(curMessage) + .build()); + } catch (Throwable t) { + this.processThrowableWhenWriteMessage(t, ctx, request, curMessageExt); + messageIterator.forEachRemaining(messageExt -> + this.processThrowableWhenWriteMessage(t, ctx, request, messageExt)); + return; + } + } + } + break; + case POLLING_FULL: + streamObserver.onNext(ReceiveMessageResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.TOO_MANY_REQUESTS, "polling full")) + .build()); + break; + case NO_NEW_MSG: + case POLLING_NOT_FOUND: + default: + streamObserver.onNext(ReceiveMessageResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MESSAGE_NOT_FOUND, "no new message")) + .build()); + break; + } + } catch (Throwable t) { + writeResponseWithErrorIgnore( + ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(t)).build()); + } finally { + onComplete(); + } + } + + protected Message convertToMessage(MessageExt messageExt) { + return GrpcConverter.getInstance().buildMessage(messageExt); + } + + protected void processThrowableWhenWriteMessage(Throwable throwable, + ProxyContext ctx, ReceiveMessageRequest request, MessageExt messageExt) { + + String handle = messageExt.getProperty(MessageConst.PROPERTY_POP_CK); + if (handle == null) { + return; + } + + this.messagingProcessor.changeInvisibleTime( + ctx, + ReceiptHandle.decode(handle), + messageExt.getMsgId(), + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()), + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getMessageQueue().getTopic()), + NACK_INVISIBLE_TIME + ); + } + + public void writeAndComplete(ProxyContext ctx, Code code, String message) { + writeResponseWithErrorIgnore( + ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(code, message)).build()); + onComplete(); + } + + public void writeAndComplete(ProxyContext ctx, ReceiveMessageRequest request, Throwable throwable) { + writeResponseWithErrorIgnore( + ReceiveMessageResponse.newBuilder().setStatus(ResponseBuilder.getInstance().buildStatus(throwable)).build()); + onComplete(); + } + + protected void writeResponseWithErrorIgnore(ReceiveMessageResponse response) { + try { + ResponseWriter.getInstance().writeResponse(streamObserver, response); + } catch (Exception e) { + log.error("err when write receive message response", e); + } + } + + protected void onComplete() { + writeResponseWithErrorIgnore(ReceiveMessageResponse.newBuilder() + .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .build()); + try { + streamObserver.onCompleted(); + } catch (Exception e) { + log.error("err when complete receive message response", e); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivity.java new file mode 100644 index 0000000000..6e1c85cff4 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivity.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class ForwardMessageToDLQActivity extends AbstractMessingActivity { + + public ForwardMessageToDLQActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + public CompletableFuture forwardMessageToDeadLetterQueue(ProxyContext ctx, + ForwardMessageToDeadLetterQueueRequest request) { + CompletableFuture future = new CompletableFuture<>(); + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); + ReceiptHandle receiptHandle = ReceiptHandle.decode(request.getReceiptHandle()); + + return this.messagingProcessor.forwardMessageToDeadLetterQueue( + ctx, + receiptHandle, + request.getMessageId(), + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()), + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()) + ).thenApply(result -> convertToForwardMessageToDeadLetterQueueResponse(ctx, result)); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected ForwardMessageToDeadLetterQueueResponse convertToForwardMessageToDeadLetterQueueResponse(ProxyContext ctx, + RemotingCommand result) { + return ForwardMessageToDeadLetterQueueResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(result.getCode(), result.getRemark())) + .build(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java new file mode 100644 index 0000000000..87a8fedec8 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Encoding; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.SendResultEntry; +import com.google.common.collect.Maps; +import com.google.common.hash.Hashing; +import com.google.protobuf.ByteString; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Durations; +import com.google.protobuf.util.Timestamps; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcValidator; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.QueueSelector; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; + +public class SendMessageActivity extends AbstractMessingActivity { + + public SendMessageActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + public CompletableFuture sendMessage(ProxyContext ctx, SendMessageRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + if (request.getMessagesCount() <= 0) { + throw new GrpcProxyException(Code.MESSAGE_CORRUPTED, "no message to send"); + } + + List messageList = request.getMessagesList(); + apache.rocketmq.v2.Message message = messageList.get(0); + Resource topic = message.getTopic(); + validateTopic(topic); + + future = this.messagingProcessor.sendMessage( + ctx, + new SendMessageQueueSelector(request), + GrpcConverter.getInstance().wrapResourceWithNamespace(topic), + buildSysFlag(message), + buildMessage(ctx, request.getMessagesList(), topic) + ).thenApply(result -> convertToSendMessageResponse(ctx, request, result)); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected List buildMessage(ProxyContext context, List protoMessageList, + Resource topic) { + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(topic); + List messageExtList = new ArrayList<>(); + for (apache.rocketmq.v2.Message protoMessage : protoMessageList) { + if (!protoMessage.getTopic().equals(topic)) { + throw new GrpcProxyException(Code.MESSAGE_CORRUPTED, "topic in message is not same"); + } + // here use topicName as producerGroup for transactional checker. + messageExtList.add(buildMessage(context, protoMessage, topicName)); + } + return messageExtList; + } + + protected Message buildMessage(ProxyContext context, apache.rocketmq.v2.Message protoMessage, String producerGroup) { + String topicName = GrpcConverter.getInstance().wrapResourceWithNamespace(protoMessage.getTopic()); + + validateMessageBodySize(protoMessage.getBody()); + Message messageExt = new Message(); + messageExt.setTopic(topicName); + messageExt.setBody(protoMessage.getBody().toByteArray()); + Map messageProperty = this.buildMessageProperty(context, protoMessage, producerGroup); + + MessageAccessor.setProperties(messageExt, messageProperty); + return messageExt; + } + + protected int buildSysFlag(apache.rocketmq.v2.Message protoMessage) { + // sysFlag (body encoding & message type) + int sysFlag = 0; + Encoding bodyEncoding = protoMessage.getSystemProperties().getBodyEncoding(); + if (bodyEncoding.equals(Encoding.GZIP)) { + sysFlag |= MessageSysFlag.COMPRESSED_FLAG; + } + // transaction + MessageType messageType = protoMessage.getSystemProperties().getMessageType(); + if (messageType.equals(MessageType.TRANSACTION)) { + sysFlag |= MessageSysFlag.TRANSACTION_PREPARED_TYPE; + } + return sysFlag; + } + + protected void validateMessageBodySize(ByteString body) { + int max = ConfigurationManager.getProxyConfig().getMaxMessageSize(); + if (max <= 0) { + return; + } + if (body.size() > max) { + throw new GrpcProxyException(Code.MESSAGE_BODY_TOO_LARGE, "message body cannot exceed the max " + max); + } + } + + protected void validateMessageKey(String key) { + if (StringUtils.isNotEmpty(key)) { + if (StringUtils.isBlank(key)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_KEY, "key cannot be the char sequence of whitespace"); + } + if (GrpcValidator.getInstance().containControlCharacter(key)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_KEY, "key cannot contain control character"); + } + } + } + + protected void validateMessageGroup(String messageGroup) { + if (StringUtils.isNotEmpty(messageGroup)) { + if (StringUtils.isBlank(messageGroup)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, "message group cannot be the char sequence of whitespace"); + } + int maxSize = ConfigurationManager.getProxyConfig().getMaxMessageGroupSize(); + if (maxSize <= 0) { + return; + } + if (messageGroup.getBytes(StandardCharsets.UTF_8).length >= maxSize) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, "message group exceed the max size " + maxSize); + } + if (GrpcValidator.getInstance().containControlCharacter(messageGroup)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_GROUP, "message group cannot contain control character"); + } + } + } + + protected void validateDelayTime(long deliveryTimestampMs) { + long maxDelay = ConfigurationManager.getProxyConfig().getMaxDelayTimeMills(); + if (maxDelay <= 0) { + return; + } + if (deliveryTimestampMs - System.currentTimeMillis() > maxDelay) { + throw new GrpcProxyException(Code.ILLEGAL_DELIVERY_TIME, "the max delay time of message is too large, max is " + maxDelay); + } + } + + protected void validateTransactionRecoverySecond(long transactionRecoverySecond) { + long maxTransactionRecoverySecond = ConfigurationManager.getProxyConfig().getMaxTransactionRecoverySecond(); + if (maxTransactionRecoverySecond <= 0) { + return; + } + if (transactionRecoverySecond > maxTransactionRecoverySecond) { + throw new GrpcProxyException(Code.BAD_REQUEST, "the max transaction recovery time of message is too large, max is " + maxTransactionRecoverySecond); + } + } + + protected Map buildMessageProperty(ProxyContext context, apache.rocketmq.v2.Message message, String producerGroup) { + long userPropertySize = 0; + ProxyConfig config = ConfigurationManager.getProxyConfig(); + org.apache.rocketmq.common.message.Message messageWithHeader = new org.apache.rocketmq.common.message.Message(); + // set user properties + Map userProperties = message.getUserPropertiesMap(); + if (userProperties.size() > config.getUserPropertyMaxNum()) { + throw new GrpcProxyException(Code.MESSAGE_PROPERTIES_TOO_LARGE, "too many user properties, max is " + config.getUserPropertyMaxNum()); + } + for (Map.Entry userPropertiesEntry : userProperties.entrySet()) { + if (MessageConst.STRING_HASH_SET.contains(userPropertiesEntry.getKey())) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, "property is used by system: " + userPropertiesEntry.getKey()); + } + userPropertySize += userPropertiesEntry.getKey().getBytes(StandardCharsets.UTF_8).length; + userPropertySize += userPropertiesEntry.getValue().getBytes(StandardCharsets.UTF_8).length; + } + MessageAccessor.setProperties(messageWithHeader, Maps.newHashMap(userProperties)); + + // set tag + String tag = message.getSystemProperties().getTag(); + GrpcValidator.getInstance().validateTag(tag); + messageWithHeader.setTags(tag); + userPropertySize += tag.getBytes(StandardCharsets.UTF_8).length; + + // set keys + List keysList = message.getSystemProperties().getKeysList(); + for (String key : keysList) { + validateMessageKey(key); + userPropertySize += key.getBytes(StandardCharsets.UTF_8).length; + } + if (keysList.size() > 0) { + messageWithHeader.setKeys(keysList); + } + + if (userPropertySize > config.getMaxUserPropertySize()) { + throw new GrpcProxyException(Code.MESSAGE_PROPERTIES_TOO_LARGE, "the total size of user property is too large, max is " + config.getMaxUserPropertySize()); + } + + // set message id + String messageId = message.getSystemProperties().getMessageId(); + if (StringUtils.isBlank(messageId)) { + throw new GrpcProxyException(Code.ILLEGAL_MESSAGE_ID, "message id cannot be empty"); + } + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, messageId); + + // set transaction property + MessageType messageType = message.getSystemProperties().getMessageType(); + if (messageType.equals(MessageType.TRANSACTION)) { + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + + if (message.getSystemProperties().hasOrphanedTransactionRecoveryDuration()) { + long transactionRecoverySecond = Durations.toSeconds(message.getSystemProperties().getOrphanedTransactionRecoveryDuration()); + validateTransactionRecoverySecond(transactionRecoverySecond); + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, + String.valueOf(transactionRecoverySecond)); + } + } + + // set delay level or deliver timestamp + fillDelayMessageProperty(message, messageWithHeader); + + // set reconsume times + int reconsumeTimes = message.getSystemProperties().getDeliveryAttempt(); + MessageAccessor.setReconsumeTime(messageWithHeader, String.valueOf(reconsumeTimes)); + // set producer group + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_PRODUCER_GROUP, producerGroup); + // set message group + String messageGroup = message.getSystemProperties().getMessageGroup(); + if (StringUtils.isNotEmpty(messageGroup)) { + validateMessageGroup(messageGroup); + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_SHARDING_KEY, messageGroup); + } + // set trace context + String traceContext = message.getSystemProperties().getTraceContext(); + if (!traceContext.isEmpty()) { + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TRACE_CONTEXT, traceContext); + } + + String bornHost = message.getSystemProperties().getBornHost(); + if (StringUtils.isBlank(bornHost)) { + bornHost = context.getRemoteAddress(); + } + if (StringUtils.isNotBlank(bornHost)) { + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_BORN_HOST, bornHost); + } + + return messageWithHeader.getProperties(); + } + + protected void fillDelayMessageProperty(apache.rocketmq.v2.Message message, org.apache.rocketmq.common.message.Message messageWithHeader) { + if (message.getSystemProperties().hasDeliveryTimestamp()) { + Timestamp deliveryTimestamp = message.getSystemProperties().getDeliveryTimestamp(); + long deliveryTimestampMs = Timestamps.toMillis(deliveryTimestamp); + validateDelayTime(deliveryTimestampMs); + + ProxyConfig config = ConfigurationManager.getProxyConfig(); + if (config.isUseDelayLevel()) { + int delayLevel = config.computeDelayLevel(deliveryTimestampMs); + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_DELAY_TIME_LEVEL, String.valueOf(delayLevel)); + } + + String timestampString = String.valueOf(deliveryTimestampMs); + MessageAccessor.putProperty(messageWithHeader, MessageConst.PROPERTY_TIMER_DELIVER_MS, timestampString); + } + } + + protected SendMessageResponse convertToSendMessageResponse(ProxyContext ctx, SendMessageRequest request, + List resultList) { + SendMessageResponse.Builder builder = SendMessageResponse.newBuilder(); + + Set responseCodes = new HashSet<>(); + for (SendResult result : resultList) { + SendResultEntry resultEntry; + switch (result.getSendStatus()) { + case FLUSH_DISK_TIMEOUT: + resultEntry = SendResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.MASTER_PERSISTENCE_TIMEOUT, "send message failed, sendStatus=" + result.getSendStatus())) + .build(); + break; + case FLUSH_SLAVE_TIMEOUT: + resultEntry = SendResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.SLAVE_PERSISTENCE_TIMEOUT, "send message failed, sendStatus=" + result.getSendStatus())) + .build(); + break; + case SLAVE_NOT_AVAILABLE: + resultEntry = SendResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.HA_NOT_AVAILABLE, "send message failed, sendStatus=" + result.getSendStatus())) + .build(); + break; + case SEND_OK: + resultEntry = SendResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .setOffset(result.getQueueOffset()) + .setMessageId(StringUtils.defaultString(result.getMsgId())) + .setTransactionId(StringUtils.defaultString(result.getTransactionId())) + .build(); + break; + default: + resultEntry = SendResultEntry.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "send message failed, sendStatus=" + result.getSendStatus())) + .build(); + break; + } + builder.addEntries(resultEntry); + responseCodes.add(resultEntry.getStatus().getCode()); + } + if (responseCodes.size() > 1) { + builder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); + } else if (responseCodes.size() == 1) { + Code code = responseCodes.stream().findAny().get(); + builder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); + } else { + builder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "send status is empty")); + } + return builder.build(); + } + + protected static class SendMessageQueueSelector implements QueueSelector { + + private final SendMessageRequest request; + + public SendMessageQueueSelector(SendMessageRequest request) { + this.request = request; + } + + @Override + public AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView) { + try { + apache.rocketmq.v2.Message message = request.getMessages(0); + String shardingKey = null; + if (request.getMessagesCount() == 1) { + shardingKey = message.getSystemProperties().getMessageGroup(); + } + AddressableMessageQueue targetMessageQueue; + if (StringUtils.isNotEmpty(shardingKey)) { + // With shardingKey + List writeQueues = messageQueueView.getWriteSelector().getQueues(); + int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size()); + targetMessageQueue = writeQueues.get(bucket); + } else { + targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false); + } + return targetMessageQueue; + } catch (Exception e) { + return null; + } + } + } + +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java new file mode 100644 index 0000000000..c5cf8a3e05 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivity.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.route; + +import apache.rocketmq.v2.Address; +import apache.rocketmq.v2.AddressScheme; +import apache.rocketmq.v2.Assignment; +import apache.rocketmq.v2.Broker; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Endpoints; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.Permission; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.Resource; +import com.google.common.net.HostAndPort; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; + +public class RouteActivity extends AbstractMessingActivity { + + public RouteActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + public CompletableFuture queryRoute(ProxyContext ctx, QueryRouteRequest request) { + CompletableFuture future = new CompletableFuture<>(); + try { + validateTopic(request.getTopic()); + List addressList = this.convertToAddressList(request.getEndpoints()); + + ProxyTopicRouteData proxyTopicRouteData = this.messagingProcessor.getTopicRouteDataForProxy( + ctx, + addressList, + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); + + List messageQueueList = new ArrayList<>(); + Map> brokerMap = buildBrokerMap(proxyTopicRouteData.getBrokerDatas()); + + for (QueueData queueData : proxyTopicRouteData.getQueueDatas()) { + String brokerName = queueData.getBrokerName(); + Map brokerIdMap = brokerMap.get(brokerName); + if (brokerIdMap == null) { + break; + } + for (Broker broker : brokerIdMap.values()) { + messageQueueList.addAll(this.genMessageQueueFromQueueData(queueData, request.getTopic(), broker)); + } + } + + QueryRouteResponse response = QueryRouteResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .addAllMessageQueues(messageQueueList) + .build(); + future.complete(response); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture queryAssignment(ProxyContext ctx, + QueryAssignmentRequest request) { + CompletableFuture future = new CompletableFuture<>(); + + try { + validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); + List addressList = this.convertToAddressList(request.getEndpoints()); + + ProxyTopicRouteData proxyTopicRouteData = this.messagingProcessor.getTopicRouteDataForProxy( + ctx, + addressList, + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); + + List assignments = new ArrayList<>(); + Map> brokerMap = buildBrokerMap(proxyTopicRouteData.getBrokerDatas()); + for (QueueData queueData : proxyTopicRouteData.getQueueDatas()) { + if (PermName.isReadable(queueData.getPerm()) && queueData.getReadQueueNums() > 0) { + Map brokerIdMap = brokerMap.get(queueData.getBrokerName()); + if (brokerIdMap != null) { + Broker broker = brokerIdMap.get(MixAll.MASTER_ID); + MessageQueue defaultMessageQueue = MessageQueue.newBuilder() + .setTopic(request.getTopic()) + .setId(-1) + .setPermission(this.convertToPermission(queueData.getPerm())) + .setBroker(broker) + .build(); + + assignments.add(Assignment.newBuilder() + .setMessageQueue(defaultMessageQueue) + .build()); + + } + } + } + + QueryAssignmentResponse response; + if (assignments.isEmpty()) { + response = QueryAssignmentResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.FORBIDDEN, "no readable queue")) + .build(); + } else { + response = QueryAssignmentResponse.newBuilder() + .addAllAssignments(assignments) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build(); + } + future.complete(response); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected Permission convertToPermission(int perm) { + boolean isReadable = PermName.isReadable(perm); + boolean isWriteable = PermName.isWriteable(perm); + if (isReadable && isWriteable) { + return Permission.READ_WRITE; + } + if (isReadable) { + return Permission.READ; + } + if (isWriteable) { + return Permission.WRITE; + } + return Permission.NONE; + } + + protected List convertToAddressList(Endpoints endpoints) { + int port = ConfigurationManager.getProxyConfig().getGrpcServerPort(); + List addressList = new ArrayList<>(); + for (Address address : endpoints.getAddressesList()) { + addressList.add(new org.apache.rocketmq.proxy.common.Address( + org.apache.rocketmq.proxy.common.Address.AddressScheme.valueOf(endpoints.getScheme().name()), + HostAndPort.fromParts(address.getHost(), port)) + ); + } + return addressList; + } + + protected Map> buildBrokerMap( + List brokerDataList) { + Map> brokerMap = new HashMap<>(); + for (ProxyTopicRouteData.ProxyBrokerData brokerData : brokerDataList) { + Map brokerIdMap = new HashMap<>(); + String brokerName = brokerData.getBrokerName(); + for (Map.Entry> entry : brokerData.getBrokerAddrs().entrySet()) { + Long brokerId = entry.getKey(); + List

addressList = new ArrayList<>(); + AddressScheme addressScheme = AddressScheme.IPv4; + for (org.apache.rocketmq.proxy.common.Address address : entry.getValue()) { + addressScheme = AddressScheme.valueOf(address.getAddressScheme().name()); + addressList.add(Address.newBuilder() + .setHost(address.getHostAndPort().getHost()) + .setPort(address.getHostAndPort().getPort()) + .build()); + } + + Broker broker = Broker.newBuilder() + .setName(brokerName) + .setId(Math.toIntExact(brokerId)) + .setEndpoints(Endpoints.newBuilder() + .setScheme(addressScheme) + .addAllAddresses(addressList) + .build()) + .build(); + + brokerIdMap.put(brokerId, broker); + } + brokerMap.put(brokerName, brokerIdMap); + } + return brokerMap; + } + + protected List genMessageQueueFromQueueData(QueueData queueData, Resource topic, Broker broker) { + List messageQueueList = new ArrayList<>(); + + int r = 0; + int w = 0; + int rw = 0; + if (PermName.isWriteable(queueData.getPerm()) && PermName.isReadable(queueData.getPerm())) { + rw = Math.min(queueData.getWriteQueueNums(), queueData.getReadQueueNums()); + r = queueData.getReadQueueNums() - rw; + w = queueData.getWriteQueueNums() - rw; + } else if (PermName.isWriteable(queueData.getPerm())) { + w = queueData.getWriteQueueNums(); + } else if (PermName.isReadable(queueData.getPerm())) { + r = queueData.getReadQueueNums(); + } + + // r here means readOnly queue nums, w means writeOnly queue nums, while rw means both readable and writable queue nums. + int queueIdIndex = 0; + for (int i = 0; i < r; i++) { + MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic) + .setId(queueIdIndex++) + .setPermission(Permission.READ) + .build(); + messageQueueList.add(messageQueue); + } + + for (int i = 0; i < w; i++) { + MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic) + .setId(queueIdIndex++) + .setPermission(Permission.WRITE) + .build(); + messageQueueList.add(messageQueue); + } + + for (int i = 0; i < rw; i++) { + MessageQueue messageQueue = MessageQueue.newBuilder().setBroker(broker).setTopic(topic) + .setId(queueIdIndex++) + .setPermission(Permission.READ_WRITE) + .build(); + messageQueueList.add(messageQueue); + } + + return messageQueueList; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java new file mode 100644 index 0000000000..e65cf2eb4f --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivity.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.grpc.v2.transaction; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.TransactionResolution; +import apache.rocketmq.v2.TransactionSource; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.TransactionStatus; + +public class EndTransactionActivity extends AbstractMessingActivity { + + public EndTransactionActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + public CompletableFuture endTransaction(ProxyContext ctx, EndTransactionRequest request) { + CompletableFuture future = new CompletableFuture<>(); + try { + validateTopic(request.getTopic()); + if (StringUtils.isBlank(request.getTransactionId())) { + throw new GrpcProxyException(Code.INVALID_TRANSACTION_ID, "transaction id cannot be empty"); + } + + TransactionStatus transactionStatus = TransactionStatus.UNKNOWN; + TransactionResolution transactionResolution = request.getResolution(); + switch (transactionResolution) { + case COMMIT: + transactionStatus = TransactionStatus.COMMIT; + break; + case ROLLBACK: + transactionStatus = TransactionStatus.ROLLBACK; + break; + default: + break; + } + future = this.messagingProcessor.endTransaction( + ctx, + request.getTransactionId(), + request.getMessageId(), + GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()), + transactionStatus, + request.getSource().equals(TransactionSource.SOURCE_SERVER_CHECK)) + .thenApply(r -> EndTransactionResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java new file mode 100644 index 0000000000..c223eb4783 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.service.ServiceManager; + +public abstract class AbstractProcessor extends AbstractStartAndShutdown { + + protected MessagingProcessor messagingProcessor; + protected ServiceManager serviceManager; + + public AbstractProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager) { + this.messagingProcessor = messagingProcessor; + this.serviceManager = serviceManager; + } + + protected void validateReceiptHandle(ReceiptHandle handle) { + if (handle.isExpired()) { + throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); + } + } + + protected TopicMessageType parseFromMessageExt(Message message) { + String isTrans = message.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + String isTransValue = "true"; + if (isTransValue.equals(isTrans)) { + return TopicMessageType.TRANSACTION; + } else if (message.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL) != null + || message.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS) != null + || message.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC) != null) { + return TopicMessageType.DELAY; + } else if (message.getProperty(MessageConst.PROPERTY_SHARDING_KEY) != null) { + return TopicMessageType.FIFO; + } else { + return TopicMessageType.NORMAL; + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java new file mode 100644 index 0000000000..9225289822 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ClientProcessor.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import io.netty.channel.Channel; +import java.util.Set; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.broker.client.ProducerChangeListener; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.ServiceManager; + +public class ClientProcessor extends AbstractProcessor { + + public ClientProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager) { + super(messagingProcessor, serviceManager); + } + + public void registerProducer( + ProxyContext ctx, + String producerGroup, + ClientChannelInfo clientChannelInfo + ) { + this.serviceManager.getProducerManager().registerProducer(producerGroup, clientChannelInfo); + } + + public void unRegisterProducer( + ProxyContext ctx, + String producerGroup, + ClientChannelInfo clientChannelInfo + ) { + this.serviceManager.getProducerManager().unregisterProducer(producerGroup, clientChannelInfo); + } + + public Channel findProducerChannel( + ProxyContext ctx, + String producerGroup, + String clientId + ) { + return this.serviceManager.getProducerManager().findChannel(clientId); + } + + public void registerProducerChangeListener(ProducerChangeListener listener) { + this.serviceManager.getProducerManager().appendProducerChangeListener(listener); + } + + public void registerConsumer( + ProxyContext ctx, + String consumerGroup, + ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, + MessageModel messageModel, + ConsumeFromWhere consumeFromWhere, + Set subList, + boolean updateSubscription + ) { + this.serviceManager.getConsumerManager().registerConsumer( + consumerGroup, + clientChannelInfo, + consumeType, + messageModel, + consumeFromWhere, + subList, + false, + updateSubscription); + } + + public ClientChannelInfo findConsumerChannel( + ProxyContext ctx, + String consumerGroup, + String clientId + ) { + return this.serviceManager.getConsumerManager().findChannel(consumerGroup, clientId); + } + + public void unRegisterConsumer( + ProxyContext ctx, + String consumerGroup, + ClientChannelInfo clientChannelInfo + ) { + this.serviceManager.getConsumerManager().unregisterConsumer(consumerGroup, clientChannelInfo, false); + } + + public void registerConsumerIdsChangeListener(ConsumerIdsChangeListener listener) { + this.serviceManager.getConsumerManager().appendConsumerIdsChangeListener(listener); + } + + public ConsumerGroupInfo getConsumerGroupInfo(String consumerGroup) { + return this.serviceManager.getConsumerManager().getConsumerGroupInfo(consumerGroup); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java new file mode 100644 index 0000000000..07ef35089b --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArraySet; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.common.utils.FutureUtils; +import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; + +public class ConsumerProcessor extends AbstractProcessor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final ExecutorService executor; + + public ConsumerProcessor(MessagingProcessor messagingProcessor, ServiceManager serviceManager, + ExecutorService executor) { + super(messagingProcessor, serviceManager); + this.executor = executor; + } + + public CompletableFuture popMessage( + ProxyContext ctx, + QueueSelector queueSelector, + String consumerGroup, + String topic, + int maxMsgNums, + long invisibleTime, + long pollTime, + int initMode, + SubscriptionData subscriptionData, + boolean fifo, + PopMessageResultFilter popMessageResultFilter, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue messageQueue = queueSelector.select(ctx, this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(topic)); + if (messageQueue == null) { + throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no readable queue"); + } + + if (maxMsgNums > ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST) { + log.warn("change maxNums from {} to {} for pop request, with info: topic:{}, group:{}", + maxMsgNums, ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST, topic, consumerGroup); + maxMsgNums = ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST; + } + + PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(topic); + requestHeader.setQueueId(messageQueue.getQueueId()); + requestHeader.setMaxMsgNums(maxMsgNums); + requestHeader.setInvisibleTime(invisibleTime); + requestHeader.setPollTime(pollTime); + requestHeader.setInitMode(initMode); + requestHeader.setExpType(subscriptionData.getExpressionType()); + requestHeader.setExp(subscriptionData.getSubString()); + requestHeader.setOrder(fifo); + + future = this.serviceManager.getMessageService().popMessage( + ctx, + messageQueue, + requestHeader, + timeoutMillis) + .thenApplyAsync(popResult -> { + if (PopStatus.FOUND.equals(popResult.getPopStatus()) && + popResult.getMsgFoundList() != null && + !popResult.getMsgFoundList().isEmpty() && + popMessageResultFilter != null) { + + List messageExtList = new ArrayList<>(); + for (MessageExt messageExt : popResult.getMsgFoundList()) { + try { + String handleString = createHandle(messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getCommitLogOffset()); + if (handleString == null) { + log.error("[BUG] pop message from broker but handle is empty. requestHeader:{}, msg:{}", requestHeader, messageExt); + messageExtList.add(messageExt); + continue; + } + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, handleString); + + PopMessageResultFilter.FilterResult filterResult = + popMessageResultFilter.filterMessage(ctx, consumerGroup, subscriptionData, messageExt); + switch (filterResult) { + case NO_MATCH: + this.messagingProcessor.ackMessage( + ctx, + ReceiptHandle.decode(handleString), + messageExt.getMsgId(), + consumerGroup, + topic, + MessagingProcessor.DEFAULT_TIMEOUT_MILLS); + break; + case TO_DLQ: + this.messagingProcessor.forwardMessageToDeadLetterQueue( + ctx, + ReceiptHandle.decode(handleString), + messageExt.getMsgId(), + consumerGroup, + topic, + MessagingProcessor.DEFAULT_TIMEOUT_MILLS); + break; + case MATCH: + default: + messageExtList.add(messageExt); + break; + } + } catch (Throwable t) { + log.error("process filterMessage failed. requestHeader:{}, msg:{}", requestHeader, messageExt, t); + messageExtList.add(messageExt); + } + } + popResult.setMsgFoundList(messageExtList); + } + return popResult; + }, this.executor); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture ackMessage( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String consumerGroup, + String topic, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.validateReceiptHandle(handle); + + AckMessageRequestHeader ackMessageRequestHeader = new AckMessageRequestHeader(); + ackMessageRequestHeader.setConsumerGroup(consumerGroup); + ackMessageRequestHeader.setTopic(handle.getRealTopic(topic, consumerGroup)); + ackMessageRequestHeader.setQueueId(handle.getQueueId()); + ackMessageRequestHeader.setExtraInfo(handle.getReceiptHandle()); + ackMessageRequestHeader.setOffset(handle.getOffset()); + + future = this.serviceManager.getMessageService().ackMessage( + ctx, + handle, + messageId, + ackMessageRequestHeader, + timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, + String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.validateReceiptHandle(handle); + + ChangeInvisibleTimeRequestHeader changeInvisibleTimeRequestHeader = new ChangeInvisibleTimeRequestHeader(); + changeInvisibleTimeRequestHeader.setConsumerGroup(groupName); + changeInvisibleTimeRequestHeader.setTopic(handle.getRealTopic(topicName, groupName)); + changeInvisibleTimeRequestHeader.setQueueId(handle.getQueueId()); + changeInvisibleTimeRequestHeader.setExtraInfo(handle.getReceiptHandle()); + changeInvisibleTimeRequestHeader.setOffset(handle.getOffset()); + changeInvisibleTimeRequestHeader.setInvisibleTime(invisibleTime); + long commitLogOffset = handle.getCommitLogOffset(); + + future = this.serviceManager.getMessageService().changeInvisibleTime( + ctx, + handle, + messageId, + changeInvisibleTimeRequestHeader, + timeoutMillis) + .thenApplyAsync(ackResult -> { + if (StringUtils.isNotBlank(ackResult.getExtraInfo())) { + AckResult result = new AckResult(); + result.setStatus(ackResult.getStatus()); + result.setPopTime(result.getPopTime()); + result.setExtraInfo(createHandle(ackResult.getExtraInfo(), commitLogOffset)); + return result; + } else { + return ackResult; + } + }, this.executor); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + protected String createHandle(String handleString, long commitLogOffset) { + if (handleString == null) { + return null; + } + return handleString + MessageConst.KEY_SEPARATOR + commitLogOffset; + } + + public CompletableFuture pullMessage(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup, + long queueOffset, int maxMsgNums, int sysFlag, long commitOffset, + long suspendTimeoutMillis, SubscriptionData subscriptionData, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(messageQueue); + PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + requestHeader.setQueueOffset(queueOffset); + requestHeader.setMaxMsgNums(maxMsgNums); + requestHeader.setSysFlag(sysFlag); + requestHeader.setCommitOffset(commitOffset); + requestHeader.setSuspendTimeoutMillis(suspendTimeoutMillis); + requestHeader.setSubscription(subscriptionData.getSubString()); + requestHeader.setExpressionType(subscriptionData.getExpressionType()); + future = serviceManager.getMessageService().pullMessage(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture updateConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long commitOffset, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(messageQueue); + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + requestHeader.setCommitOffset(commitOffset); + future = serviceManager.getMessageService().updateConsumerOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(messageQueue); + QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + future = serviceManager.getMessageService().queryConsumerOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture> lockBatchMQ(ProxyContext ctx, Set mqSet, + String consumerGroup, String clientId, long timeoutMillis) { + CompletableFuture> future = new CompletableFuture<>(); + Set successSet = new CopyOnWriteArraySet<>(); + Set addressableMessageQueueSet = buildAddressableSet(mqSet); + Map> messageQueueSetMap = buildAddressableMapByBrokerName(addressableMessageQueueSet); + List>> futureList = new ArrayList<>(); + messageQueueSetMap.forEach((k, v) -> { + LockBatchRequestBody requestBody = new LockBatchRequestBody(); + requestBody.setConsumerGroup(consumerGroup); + requestBody.setClientId(clientId); + requestBody.setMqSet(v.stream().map(AddressableMessageQueue::getMessageQueue).collect(Collectors.toSet())); + CompletableFuture> future0 = new CompletableFuture<>(); + try { + future0 = serviceManager.getMessageService().lockBatchMQ(ctx, v.get(0), requestBody, timeoutMillis); + future0.thenAccept(successSet::addAll); + } catch (Throwable t) { + future0.completeExceptionally(t); + } + futureList.add(FutureUtils.addExecutor(future0, this.executor)); + }); + CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((v, t) -> { + if (t != null) { + log.error("LockBatchMQ failed", t); + } + future.complete(successSet); + }); + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture unlockBatchMQ(ProxyContext ctx, Set mqSet, + String consumerGroup, String clientId, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + Set addressableMessageQueueSet = buildAddressableSet(mqSet); + Map> messageQueueSetMap = buildAddressableMapByBrokerName(addressableMessageQueueSet); + List> futureList = new ArrayList<>(); + messageQueueSetMap.forEach((k, v) -> { + UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); + requestBody.setConsumerGroup(consumerGroup); + requestBody.setClientId(clientId); + requestBody.setMqSet(v.stream().map(AddressableMessageQueue::getMessageQueue).collect(Collectors.toSet())); + CompletableFuture future0 = new CompletableFuture<>(); + try { + future0 = serviceManager.getMessageService().unlockBatchMQ(ctx, v.get(0), requestBody, timeoutMillis); + future0.complete(null); + } catch (Throwable t) { + future0.completeExceptionally(t); + } + futureList.add(FutureUtils.addExecutor(future0, this.executor)); + }); + CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).whenComplete((v, t) -> { + if (t != null) { + log.error("UnlockBatchMQ failed", t); + } + future.complete(null); + }); + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture getMaxOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(messageQueue); + GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + future = serviceManager.getMessageService().getMaxOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + public CompletableFuture getMinOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + AddressableMessageQueue addressableMessageQueue = serviceManager.getTopicRouteService() + .buildAddressableMessageQueue(messageQueue); + GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader(); + requestHeader.setTopic(addressableMessageQueue.getTopic()); + requestHeader.setQueueId(addressableMessageQueue.getQueueId()); + future = serviceManager.getMessageService().getMinOffset(ctx, addressableMessageQueue, requestHeader, timeoutMillis); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + protected Set buildAddressableSet(Set mqSet) { + return mqSet.stream().map(mq -> { + try { + return serviceManager.getTopicRouteService().buildAddressableMessageQueue(mq); + } catch (Exception e) { + return null; + } + }).collect(Collectors.toSet()); + } + + protected HashMap> buildAddressableMapByBrokerName( + final Set mqSet) { + HashMap> result = new HashMap<>(); + for (AddressableMessageQueue mq : mqSet) { + List mqs = result.computeIfAbsent(mq.getBrokerName(), k -> new ArrayList<>()); + mqs.add(mq); + } + return result; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java new file mode 100644 index 0000000000..5234237a22 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import io.netty.channel.Channel; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.broker.client.ProducerChangeListener; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class DefaultMessagingProcessor extends AbstractStartAndShutdown implements MessagingProcessor { + + protected ServiceManager serviceManager; + protected ProducerProcessor producerProcessor; + protected ConsumerProcessor consumerProcessor; + protected TransactionProcessor transactionProcessor; + protected ClientProcessor clientProcessor; + + protected ThreadPoolExecutor producerProcessorExecutor; + protected ThreadPoolExecutor consumerProcessorExecutor; + + protected DefaultMessagingProcessor(ServiceManager serviceManager) { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + this.producerProcessorExecutor = ThreadPoolMonitor.createAndMonitor( + proxyConfig.getProducerProcessorThreadPoolNums(), + proxyConfig.getProducerProcessorThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "ProducerProcessorExecutor", + proxyConfig.getProducerProcessorThreadPoolQueueCapacity() + ); + this.consumerProcessorExecutor = ThreadPoolMonitor.createAndMonitor( + proxyConfig.getConsumerProcessorThreadPoolNums(), + proxyConfig.getConsumerProcessorThreadPoolNums(), + 1, + TimeUnit.MINUTES, + "ConsumerProcessorExecutor", + proxyConfig.getConsumerProcessorThreadPoolQueueCapacity() + ); + + this.serviceManager = serviceManager; + this.producerProcessor = new ProducerProcessor(this, serviceManager, this.producerProcessorExecutor); + this.consumerProcessor = new ConsumerProcessor(this, serviceManager, this.consumerProcessorExecutor); + this.transactionProcessor = new TransactionProcessor(this, serviceManager); + this.clientProcessor = new ClientProcessor(this, serviceManager); + + this.init(); + } + + public static DefaultMessagingProcessor createForLocalMode(BrokerController brokerController) { + return createForLocalMode(brokerController, null); + } + + public static DefaultMessagingProcessor createForLocalMode(BrokerController brokerController, RPCHook rpcHook) { + return new DefaultMessagingProcessor(ServiceManagerFactory.createForLocalMode(brokerController, rpcHook)); + } + + public static DefaultMessagingProcessor createForClusterMode() { + return createForClusterMode(null); + } + + public static DefaultMessagingProcessor createForClusterMode(RPCHook rpcHook) { + return new DefaultMessagingProcessor(ServiceManagerFactory.createForClusterMode(rpcHook)); + } + + protected void init() { + this.appendStartAndShutdown(this.serviceManager); + this.appendShutdown(this.producerProcessorExecutor::shutdown); + this.appendShutdown(this.consumerProcessorExecutor::shutdown); + } + + @Override + public SubscriptionGroupConfig getSubscriptionGroupConfig(ProxyContext ctx, String consumerGroupName) { + return this.serviceManager.getMetadataService().getSubscriptionGroupConfig(consumerGroupName); + } + + @Override + public ProxyTopicRouteData getTopicRouteDataForProxy(ProxyContext ctx, List
requestHostAndPortList, + String topicName) throws Exception { + return this.serviceManager.getTopicRouteService().getTopicRouteForProxy(requestHostAndPortList, topicName); + } + + @Override + public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, + String producerGroup, int sysFlag, List msg, long timeoutMillis) { + return this.producerProcessor.sendMessage(ctx, queueSelector, producerGroup, sysFlag, msg, timeoutMillis); + } + + @Override + public CompletableFuture forwardMessageToDeadLetterQueue(ProxyContext ctx, ReceiptHandle handle, + String messageId, String groupName, String topicName, long timeoutMillis) { + return this.producerProcessor.forwardMessageToDeadLetterQueue(ctx, handle, messageId, groupName, topicName, timeoutMillis); + } + + @Override + public CompletableFuture endTransaction(ProxyContext ctx, String transactionId, String messageId, String producerGroup, + TransactionStatus transactionStatus, boolean fromTransactionCheck, + long timeoutMillis) { + return this.transactionProcessor.endTransaction(ctx, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, timeoutMillis); + } + + @Override + public CompletableFuture popMessage( + ProxyContext ctx, + QueueSelector queueSelector, + String consumerGroup, + String topic, + int maxMsgNums, + long invisibleTime, + long pollTime, + int initMode, + SubscriptionData subscriptionData, + boolean fifo, + PopMessageResultFilter popMessageResultFilter, + long timeoutMillis + ) { + return this.consumerProcessor.popMessage(ctx, queueSelector, consumerGroup, topic, maxMsgNums, + invisibleTime, pollTime, initMode, subscriptionData, fifo, popMessageResultFilter, timeoutMillis); + } + + @Override + public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId, + String consumerGroup, String topic, long timeoutMillis) { + return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); + } + + @Override + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, + String groupName, String topicName, long invisibleTime, long timeoutMillis) { + return this.consumerProcessor.changeInvisibleTime(ctx, handle, messageId, groupName, topicName, invisibleTime, timeoutMillis); + } + + @Override + public CompletableFuture pullMessage(ProxyContext ctx, MessageQueue messageQueue, String consumerGroup, + long queueOffset, int maxMsgNums, int sysFlag, long commitOffset, long suspendTimeoutMillis, + SubscriptionData subscriptionData, long timeoutMillis) { + return this.consumerProcessor.pullMessage(ctx, messageQueue, consumerGroup, queueOffset, maxMsgNums, + sysFlag, commitOffset, suspendTimeoutMillis, subscriptionData, timeoutMillis); + } + + @Override + public CompletableFuture updateConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long commitOffset, long timeoutMillis) { + return this.consumerProcessor.updateConsumerOffset(ctx, messageQueue, consumerGroup, commitOffset, timeoutMillis); + } + + @Override + public CompletableFuture queryConsumerOffset(ProxyContext ctx, MessageQueue messageQueue, + String consumerGroup, long timeoutMillis) { + return this.consumerProcessor.queryConsumerOffset(ctx, messageQueue, consumerGroup, timeoutMillis); + } + + @Override + public CompletableFuture> lockBatchMQ(ProxyContext ctx, Set mqSet, + String consumerGroup, String clientId, long timeoutMillis) { + return this.consumerProcessor.lockBatchMQ(ctx, mqSet, consumerGroup, clientId, timeoutMillis); + } + + @Override + public CompletableFuture unlockBatchMQ(ProxyContext ctx, Set mqSet, + String consumerGroup, + String clientId, long timeoutMillis) { + return this.consumerProcessor.unlockBatchMQ(ctx, mqSet, consumerGroup, clientId, timeoutMillis); + } + + @Override + public CompletableFuture getMaxOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) { + return this.consumerProcessor.getMaxOffset(ctx, messageQueue, timeoutMillis); + } + + @Override + public CompletableFuture getMinOffset(ProxyContext ctx, MessageQueue messageQueue, long timeoutMillis) { + return this.consumerProcessor.getMinOffset(ctx, messageQueue, timeoutMillis); + } + + @Override + public void registerProducer(ProxyContext ctx, String producerGroup, ClientChannelInfo clientChannelInfo) { + this.clientProcessor.registerProducer(ctx, producerGroup, clientChannelInfo); + } + + @Override + public void unRegisterProducer(ProxyContext ctx, String producerGroup, ClientChannelInfo clientChannelInfo) { + this.clientProcessor.unRegisterProducer(ctx, producerGroup, clientChannelInfo); + } + + @Override + public Channel findProducerChannel(ProxyContext ctx, String producerGroup, String clientId) { + return this.clientProcessor.findProducerChannel(ctx, producerGroup, clientId); + } + + @Override + public void registerProducerListener(ProducerChangeListener producerChangeListener) { + this.clientProcessor.registerProducerChangeListener(producerChangeListener); + } + + @Override + public void registerConsumer(ProxyContext ctx, String consumerGroup, ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, + Set subList, boolean updateSubscription) { + this.clientProcessor.registerConsumer(ctx, consumerGroup, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList, updateSubscription); + } + + @Override + public ClientChannelInfo findConsumerChannel(ProxyContext ctx, String consumerGroup, String clientId) { + return this.clientProcessor.findConsumerChannel(ctx, consumerGroup, clientId); + } + + @Override + public void unRegisterConsumer(ProxyContext ctx, String consumerGroup, ClientChannelInfo clientChannelInfo) { + this.clientProcessor.unRegisterConsumer(ctx, consumerGroup, clientChannelInfo); + } + + @Override + public void registerConsumerListener(ConsumerIdsChangeListener consumerIdsChangeListener) { + this.clientProcessor.registerConsumerIdsChangeListener(consumerIdsChangeListener); + } + + @Override + public ConsumerGroupInfo getConsumerGroupInfo(String consumerGroup) { + return this.clientProcessor.getConsumerGroupInfo(consumerGroup); + } + + @Override + public void addTransactionSubscription(ProxyContext ctx, String producerGroup, String topic) { + this.transactionProcessor.addTransactionSubscription(ctx, producerGroup, topic); + } + + @Override + public ProxyRelayService getProxyRelayService() { + return this.serviceManager.getProxyRelayService(); + } + + @Override + public MetadataService getMetadataService() { + return this.serviceManager.getMetadataService(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java new file mode 100644 index 0000000000..e0ae714710 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import io.netty.channel.Channel; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupInfo; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.broker.client.ProducerChangeListener; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.consumer.ConsumeFromWhere; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; +import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public interface MessagingProcessor extends StartAndShutdown { + + long DEFAULT_TIMEOUT_MILLS = Duration.ofSeconds(2).toMillis(); + + SubscriptionGroupConfig getSubscriptionGroupConfig( + ProxyContext ctx, + String consumerGroupName + ); + + ProxyTopicRouteData getTopicRouteDataForProxy( + ProxyContext ctx, + List
requestHostAndPortList, + String topicName + ) throws Exception; + + default CompletableFuture> sendMessage( + ProxyContext ctx, + QueueSelector queueSelector, + String producerGroup, + int sysFlag, + List msg + ) { + return sendMessage(ctx, queueSelector, producerGroup, sysFlag, msg, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture> sendMessage( + ProxyContext ctx, + QueueSelector queueSelector, + String producerGroup, + int sysFlag, + List msg, + long timeoutMillis + ); + + default CompletableFuture forwardMessageToDeadLetterQueue( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String groupName, + String topicName + ) { + return forwardMessageToDeadLetterQueue(ctx, handle, messageId, groupName, topicName, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture forwardMessageToDeadLetterQueue( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String groupName, + String topicName, + long timeoutMillis + ); + + default CompletableFuture endTransaction( + ProxyContext ctx, + String transactionId, + String messageId, + String producerGroup, + TransactionStatus transactionStatus, + boolean fromTransactionCheck + ) { + return endTransaction(ctx, transactionId, messageId, producerGroup, transactionStatus, fromTransactionCheck, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture endTransaction( + ProxyContext ctx, + String transactionId, + String messageId, + String producerGroup, + TransactionStatus transactionStatus, + boolean fromTransactionCheck, + long timeoutMillis + ); + + CompletableFuture popMessage( + ProxyContext ctx, + QueueSelector queueSelector, + String consumerGroup, + String topic, + int maxMsgNums, + long invisibleTime, + long pollTime, + int initMode, + SubscriptionData subscriptionData, + boolean fifo, + PopMessageResultFilter popMessageResultFilter, + long timeoutMillis + ); + + default CompletableFuture ackMessage( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String consumerGroup, + String topic + ) { + return ackMessage(ctx, handle, messageId, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture ackMessage( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String consumerGroup, + String topic, + long timeoutMillis + ); + + default CompletableFuture changeInvisibleTime( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String groupName, + String topicName, + long invisibleTime + ) { + return changeInvisibleTime(ctx, handle, messageId, groupName, topicName, invisibleTime, DEFAULT_TIMEOUT_MILLS); + } + + CompletableFuture changeInvisibleTime( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + String groupName, + String topicName, + long invisibleTime, + long timeoutMillis + ); + + CompletableFuture pullMessage( + ProxyContext ctx, + MessageQueue messageQueue, + String consumerGroup, + long queueOffset, + int maxMsgNums, + int sysFlag, + long commitOffset, + long suspendTimeoutMillis, + SubscriptionData subscriptionData, + long timeoutMillis + ); + + CompletableFuture updateConsumerOffset( + ProxyContext ctx, + MessageQueue messageQueue, + String consumerGroup, + long commitOffset, + long timeoutMillis + ); + + CompletableFuture queryConsumerOffset( + ProxyContext ctx, + MessageQueue messageQueue, + String consumerGroup, + long timeoutMillis + ); + + CompletableFuture> lockBatchMQ( + ProxyContext ctx, + Set mqSet, + String consumerGroup, + String clientId, + long timeoutMillis + ); + + CompletableFuture unlockBatchMQ( + ProxyContext ctx, + Set mqSet, + String consumerGroup, + String clientId, + long timeoutMillis + ); + + CompletableFuture getMaxOffset( + ProxyContext ctx, + MessageQueue messageQueue, + long timeoutMillis + ); + + CompletableFuture getMinOffset( + ProxyContext ctx, + MessageQueue messageQueue, + long timeoutMillis + ); + + void registerProducer( + ProxyContext ctx, + String producerGroup, + ClientChannelInfo clientChannelInfo + ); + + void unRegisterProducer( + ProxyContext ctx, + String producerGroup, + ClientChannelInfo clientChannelInfo + ); + + Channel findProducerChannel( + ProxyContext ctx, + String producerGroup, + String clientId + ); + + void registerProducerListener( + ProducerChangeListener producerChangeListener + ); + + void registerConsumer( + ProxyContext ctx, + String consumerGroup, + ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, + MessageModel messageModel, + ConsumeFromWhere consumeFromWhere, + Set subList, + boolean updateSubscription + ); + + ClientChannelInfo findConsumerChannel( + ProxyContext ctx, + String consumerGroup, + String clientId + ); + + void unRegisterConsumer( + ProxyContext ctx, + String consumerGroup, + ClientChannelInfo clientChannelInfo + ); + + void registerConsumerListener( + ConsumerIdsChangeListener consumerIdsChangeListener + ); + + ConsumerGroupInfo getConsumerGroupInfo(String consumerGroup); + + void addTransactionSubscription( + ProxyContext ctx, + String producerGroup, + String topic + ); + + ProxyRelayService getProxyRelayService(); + + MetadataService getMetadataService(); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/PopMessageResultFilter.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/PopMessageResultFilter.java new file mode 100644 index 0000000000..328ae94e30 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/PopMessageResultFilter.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.proxy.common.ProxyContext; + +public interface PopMessageResultFilter { + + enum FilterResult { + TO_DLQ, + NO_MATCH, + MATCH + } + + FilterResult filterMessage(ProxyContext ctx, String consumerGroup, SubscriptionData subscriptionData, + MessageExt messageExt); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java new file mode 100644 index 0000000000..0f47393da5 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageId; +import org.apache.rocketmq.common.protocol.NamespaceUtil; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.common.utils.FutureUtils; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.processor.validator.DefaultTopicMessageTypeValidator; +import org.apache.rocketmq.proxy.processor.validator.TopicMessageTypeValidator; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class ProducerProcessor extends AbstractProcessor { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private final ExecutorService executor; + private final TopicMessageTypeValidator topicMessageTypeValidator; + + public ProducerProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager, ExecutorService executor) { + super(messagingProcessor, serviceManager); + this.executor = executor; + this.topicMessageTypeValidator = new DefaultTopicMessageTypeValidator(); + } + + public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, + String producerGroup, int sysFlag, List messageList, long timeoutMillis) { + CompletableFuture> future = new CompletableFuture<>(); + try { + Message message = messageList.get(0); + String topic = message.getTopic(); + if (ConfigurationManager.getProxyConfig().isEnableTopicMessageTypeCheck()) { + if (topicMessageTypeValidator != null) { + // Do not check retry or dlq topic + if (!NamespaceUtil.isRetryTopic(topic) && !NamespaceUtil.isDLQTopic(topic)) { + TopicMessageType topicMessageType = serviceManager.getMetadataService().getTopicMessageType(topic); + TopicMessageType messageType = parseFromMessageExt(message); + topicMessageTypeValidator.validate(topicMessageType, messageType); + } + } + } + AddressableMessageQueue messageQueue = queueSelector.select(ctx, + this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(topic)); + if (messageQueue == null) { + throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue"); + } + + SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId()); + + future = this.serviceManager.getMessageService().sendMessage( + ctx, + messageQueue, + messageList, + requestHeader, + timeoutMillis) + .thenApplyAsync(sendResultList -> { + for (SendResult sendResult : sendResultList) { + int tranType = MessageSysFlag.getTransactionValue(requestHeader.getSysFlag()); + if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) && + tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE && + StringUtils.isNotBlank(sendResult.getTransactionId())) { + fillTransactionData(producerGroup, messageQueue, sendResult, messageList); + } + } + return sendResultList; + }, this.executor); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + + protected void fillTransactionData(String producerGroup, AddressableMessageQueue messageQueue, SendResult sendResult, List messageList) { + try { + MessageId id; + if (sendResult.getOffsetMsgId() != null) { + id = MessageDecoder.decodeMessageId(sendResult.getOffsetMsgId()); + } else { + id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); + } + this.serviceManager.getTransactionService().addTransactionDataByBrokerName( + messageQueue.getBrokerName(), + producerGroup, + sendResult.getQueueOffset(), + id.getOffset(), + sendResult.getTransactionId(), + messageList.get(0) + ); + } catch (Throwable t) { + log.warn("fillTransactionData failed. messageQueue: {}, sendResult: {}", messageQueue, sendResult, t); + } + } + + protected SendMessageRequestHeader buildSendMessageRequestHeader(List messageList, + String producerGroup, int sysFlag, int queueId) { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + + Message message = messageList.get(0); + + requestHeader.setProducerGroup(producerGroup); + requestHeader.setTopic(message.getTopic()); + requestHeader.setDefaultTopic(TopicValidator.AUTO_CREATE_TOPIC_KEY_TOPIC); + requestHeader.setDefaultTopicQueueNums(4); + requestHeader.setQueueId(queueId); + requestHeader.setSysFlag(sysFlag); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(message.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(message.getProperties())); + requestHeader.setReconsumeTimes(0); + if (messageList.size() > 1) { + requestHeader.setBatch(true); + } + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String reconsumeTimes = MessageAccessor.getReconsumeTime(message); + if (reconsumeTimes != null) { + requestHeader.setReconsumeTimes(Integer.valueOf(reconsumeTimes)); + MessageAccessor.clearProperty(message, MessageConst.PROPERTY_RECONSUME_TIME); + } + + String maxReconsumeTimes = MessageAccessor.getMaxReconsumeTimes(message); + if (maxReconsumeTimes != null) { + requestHeader.setMaxReconsumeTimes(Integer.valueOf(maxReconsumeTimes)); + MessageAccessor.clearProperty(message, MessageConst.PROPERTY_MAX_RECONSUME_TIMES); + } + } + + return requestHeader; + } + + public CompletableFuture forwardMessageToDeadLetterQueue(ProxyContext ctx, ReceiptHandle handle, + String messageId, String groupName, String topicName, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + if (handle.getCommitLogOffset() < 0) { + throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "commit log offset is empty"); + } + + ConsumerSendMsgBackRequestHeader consumerSendMsgBackRequestHeader = new ConsumerSendMsgBackRequestHeader(); + consumerSendMsgBackRequestHeader.setOffset(handle.getCommitLogOffset()); + consumerSendMsgBackRequestHeader.setGroup(groupName); + consumerSendMsgBackRequestHeader.setDelayLevel(-1); + consumerSendMsgBackRequestHeader.setOriginMsgId(messageId); + consumerSendMsgBackRequestHeader.setOriginTopic(handle.getRealTopic(topicName, groupName)); + consumerSendMsgBackRequestHeader.setMaxReconsumeTimes(0); + + future = this.serviceManager.getMessageService().sendMessageBack( + ctx, + handle, + messageId, + consumerSendMsgBackRequestHeader, + timeoutMillis + ).whenCompleteAsync((remotingCommand, t) -> { + if (t == null && remotingCommand.getCode() == ResponseCode.SUCCESS) { + this.messagingProcessor.ackMessage(ctx, handle, messageId, + groupName, topicName, timeoutMillis); + } + }, this.executor); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return FutureUtils.addExecutor(future, this.executor); + } + +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/QueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/QueueSelector.java new file mode 100644 index 0000000000..5fe0d1c38a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/QueueSelector.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; + +public interface QueueSelector { + + AddressableMessageQueue select(ProxyContext ctx, MessageQueueView messageQueueView); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java new file mode 100644 index 0000000000..dc91061fdd --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessor.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupEvent; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.subscription.RetryPolicy; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.MessageReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ReceiptHandleGroup; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ReceiptHandleProcessor extends AbstractStartAndShutdown { + protected final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected final ConcurrentMap receiptHandleGroupMap; + protected final ScheduledExecutorService scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); + protected final ExecutorService renewalWorkerService = ThreadPoolMonitor.createAndMonitor( + 2, 4, 0, TimeUnit.MILLISECONDS, + "RenewalWorkerThread_", 10000 + ); + protected final MessagingProcessor messagingProcessor; + + public ReceiptHandleProcessor(MessagingProcessor messagingProcessor) { + this.messagingProcessor = messagingProcessor; + this.messagingProcessor.registerConsumerListener(new ConsumerIdsChangeListener() { + @Override + public void handle(ConsumerGroupEvent event, String group, Object... args) { + if (ConsumerGroupEvent.CLIENT_UNREGISTER.equals(event)) { + if (args == null || args.length < 1) { + return; + } + if (args[0] instanceof ClientChannelInfo) { + ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; + clearGroup(buildKey(clientChannelInfo.getClientId(), group)); + } + } + } + + @Override + public void shutdown() { + + } + }); + this.receiptHandleGroupMap = new ConcurrentHashMap<>(); + this.init(); + } + + protected void init() { + this.appendStartAndShutdown(new StartAndShutdown() { + @Override + public void start() throws Exception { + log.info("scan for renewal start."); + scheduledExecutorService.scheduleAtFixedRate(() -> scheduleRenewTask(), 0, + ConfigurationManager.getProxyConfig().getRenewSchedulePeriodMillis(), TimeUnit.MILLISECONDS); + log.info("renewal queue has started"); + } + + @Override + public void shutdown() throws Exception { + scheduledExecutorService.shutdown(); + } + }); + } + + protected ProxyContext createContext(String actionName) { + return ProxyContext.createForInner(this.getClass().getSimpleName() + actionName); + } + + protected void scheduleRenewTask() { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + for (Map.Entry entry : receiptHandleGroupMap.entrySet()) { + String key = entry.getKey(); + Pair clientIdAndGroup = parseKey(key); + if (clientIdAndGroup == null) { + log.warn("client id and group is empty. key:{}, receiptHandleGroup:{}", key, entry.getValue()); + clearGroup(key); + continue; + } + if (clientIsOffline(clientIdAndGroup.getLeft(), clientIdAndGroup.getRight())) { + clearGroup(key); + continue; + } + + ReceiptHandleGroup group = entry.getValue(); + group.scan((msgID, handleStr, v) -> { + ReceiptHandle handle = ReceiptHandle.decode(v.getReceiptHandle()); + long now = System.currentTimeMillis(); + if (handle.getNextVisibleTime() - now > proxyConfig.getRenewAheadTimeMillis()) { + return; + } + SubscriptionGroupConfig subscriptionGroupConfig = + messagingProcessor.getMetadataService().getSubscriptionGroupConfig(v.getGroup()); + if (subscriptionGroupConfig == null) { + log.error("Group's subscriptionGroupConfig is null, group = {}", v.getGroup()); + return; + } + RetryPolicy retryPolicy = subscriptionGroupConfig.getGroupRetryPolicy().getRetryPolicy(); + renewalWorkerService.submit(() -> renewMessage(key, msgID, v, handle, retryPolicy)); + }); + } + + log.info("scan for renewal done."); + } + + protected void renewMessage(String key, String msgID, MessageReceiptHandle messageReceiptHandle, + ReceiptHandle handle, RetryPolicy retryPolicy) { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + ProxyContext context = createContext("RenewMessage"); + long current = System.currentTimeMillis(); + if (current - messageReceiptHandle.getTimestamp() < messageReceiptHandle.getExpectInvisibleTime()) { + CompletableFuture future = + messagingProcessor.changeInvisibleTime(context, handle, messageReceiptHandle.getMessageId(), + messageReceiptHandle.getGroup(), messageReceiptHandle.getTopic(), proxyConfig.getRenewSliceTimeMillis()); + future.thenAccept(ackResult -> { + if (AckStatus.OK.equals(ackResult.getStatus())) { + messageReceiptHandle.update(ackResult.getExtraInfo()); + addReceiptHandle(key, msgID, messageReceiptHandle.getOriginalReceiptHandle(), messageReceiptHandle); + } + }); + } else { + CompletableFuture future = messagingProcessor.changeInvisibleTime(context, + handle, messageReceiptHandle.getMessageId(), messageReceiptHandle.getGroup(), + messageReceiptHandle.getTopic(), retryPolicy.nextDelayDuration(messageReceiptHandle.getReconsumeTimes())); + future.thenAccept(ackResult -> { + if (AckStatus.OK.equals(ackResult.getStatus())) { + removeReceiptHandle(key, msgID, messageReceiptHandle.getOriginalReceiptHandle()); + } + }); + } + } + + protected String buildKey(String clientID, String group) { + return clientID + "%" + group; + } + + protected Pair parseKey(String key) { + String[] strs = key.split("%"); + if (strs.length < 2) { + return null; + } + return Pair.of(strs[0], strs[1]); + } + + protected boolean clientIsOffline(String clientID, String group) { + return this.messagingProcessor.findConsumerChannel(createContext("JudgeClientOnline"), group, clientID) == null; + } + + public void addReceiptHandle(String clientID, String group, String msgID, String receiptHandle, + MessageReceiptHandle messageReceiptHandle) { + this.addReceiptHandle(buildKey(clientID, group), msgID, receiptHandle, messageReceiptHandle); + } + + protected void addReceiptHandle(String key, String msgID, String receiptHandle, + MessageReceiptHandle messageReceiptHandle) { + if (key == null) { + return; + } + receiptHandleGroupMap.computeIfAbsent(key, + k -> new ReceiptHandleGroup()).put(msgID, receiptHandle, messageReceiptHandle); + } + + public MessageReceiptHandle removeReceiptHandle(String clientID, String group, String msgID, String receiptHandle) { + return this.removeReceiptHandle(buildKey(clientID, group), msgID, receiptHandle); + } + + protected MessageReceiptHandle removeReceiptHandle(String key, String msgID, String receiptHandle) { + if (key == null) { + return null; + } + AtomicReference res = new AtomicReference<>(); + receiptHandleGroupMap.computeIfPresent(key, (k, v) -> { + res.set(v.remove(msgID, receiptHandle)); + if (v.isEmpty()) { + return null; + } + return v; + }); + return res.get(); + } + + public MessageReceiptHandle removeOneReceiptHandle(String clientID, String group, String msgID) { + return removeOneReceiptHandle(buildKey(clientID, group), msgID); + } + + protected MessageReceiptHandle removeOneReceiptHandle(String key, String msgID) { + if (key == null) { + return null; + } + AtomicReference res = new AtomicReference<>(); + receiptHandleGroupMap.computeIfPresent(key, (k, v) -> { + res.set(v.removeOne(msgID)); + if (v.isEmpty()) { + return null; + } + return v; + }); + return res.get(); + } + + protected void clearGroup(String key) { + if (key == null) { + return; + } + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + ProxyContext context = createContext("ClearGroup"); + ReceiptHandleGroup handleGroup = receiptHandleGroupMap.remove(key); + if (handleGroup == null) { + return; + } + handleGroup.scan((msgID, handle, messageReceiptHandle) -> { + ReceiptHandle receiptHandle = ReceiptHandle.decode(messageReceiptHandle.getReceiptHandle()); + messagingProcessor.changeInvisibleTime( + context, + receiptHandle, + messageReceiptHandle.getMessageId(), + messageReceiptHandle.getGroup(), + messageReceiptHandle.getTopic(), + proxyConfig.getInvisibleTimeMillisWhenClear() + ); + }); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java new file mode 100644 index 0000000000..3b284cd056 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionProcessor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.transaction.EndTransactionRequestData; + +public class TransactionProcessor extends AbstractProcessor { + + public TransactionProcessor(MessagingProcessor messagingProcessor, + ServiceManager serviceManager) { + super(messagingProcessor, serviceManager); + } + + public CompletableFuture endTransaction(ProxyContext ctx, String transactionId, String messageId, String producerGroup, + TransactionStatus transactionStatus, boolean fromTransactionCheck, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + EndTransactionRequestData headerData = serviceManager.getTransactionService().genEndTransactionRequestHeader( + producerGroup, + buildCommitOrRollback(transactionStatus), + fromTransactionCheck, + messageId, + transactionId + ); + if (headerData == null) { + future.completeExceptionally(new ProxyException(ProxyExceptionCode.TRANSACTION_DATA_NOT_FOUND, "cannot found transaction data")); + return future; + } + return this.serviceManager.getMessageService().endTransactionOneway( + ctx, + headerData.getBrokerName(), + headerData.getRequestHeader(), + timeoutMillis + ); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + protected int buildCommitOrRollback(TransactionStatus transactionStatus) { + switch (transactionStatus) { + case COMMIT: + return MessageSysFlag.TRANSACTION_COMMIT_TYPE; + case ROLLBACK: + return MessageSysFlag.TRANSACTION_ROLLBACK_TYPE; + default: + return MessageSysFlag.TRANSACTION_NOT_TYPE; + } + } + + public void addTransactionSubscription(ProxyContext ctx, String producerGroup, String topic) { + this.serviceManager.getTransactionService().addTransactionSubscription(producerGroup, topic); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionStatus.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionStatus.java new file mode 100644 index 0000000000..e456a6061a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/TransactionStatus.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.processor; + +public enum TransactionStatus { + UNKNOWN, + COMMIT, + ROLLBACK +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/DefaultTopicMessageTypeValidator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/DefaultTopicMessageTypeValidator.java new file mode 100644 index 0000000000..476fbbebb7 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/DefaultTopicMessageTypeValidator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor.validator; + +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; + +public class DefaultTopicMessageTypeValidator implements TopicMessageTypeValidator { + + public void validate(TopicMessageType topicMessageType, TopicMessageType messageType) { + if (messageType.equals(TopicMessageType.UNSPECIFIED) || !messageType.equals(topicMessageType)) { + throw new ProxyException(ProxyExceptionCode.MESSAGE_PROPERTY_CONFLICT_WITH_TYPE, messageType.name() + " " + topicMessageType.name()); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/TopicMessageTypeValidator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/TopicMessageTypeValidator.java new file mode 100644 index 0000000000..137be90956 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/validator/TopicMessageTypeValidator.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor.validator; + +import org.apache.rocketmq.common.attribute.TopicMessageType; + +public interface TopicMessageTypeValidator { + /** + * Will throw {@link org.apache.rocketmq.proxy.common.ProxyException} if validate failed. + * + * @param topicMessageType Target topic + * @param messageType Message's type + */ + void validate(TopicMessageType topicMessageType, TopicMessageType messageType); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java new file mode 100644 index 0000000000..f0e0c98fb5 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerGroupEvent; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.client.ProducerChangeListener; +import org.apache.rocketmq.broker.client.ProducerGroupEvent; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.message.ClusterMessageService; +import org.apache.rocketmq.proxy.service.message.MessageService; +import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.mqclient.DoNothingClientRemotingProcessor; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.mqclient.ProxyClientRemotingProcessor; +import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.proxy.service.transaction.ClusterTransactionService; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.RPCHook; + +public class ClusterServiceManager extends AbstractStartAndShutdown implements ServiceManager { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected ClusterTransactionService clusterTransactionService; + protected ProducerManager producerManager; + protected ConsumerManager consumerManager; + protected TopicRouteService topicRouteService; + protected MessageService messageService; + protected ProxyRelayService proxyRelayService; + protected ClusterMetadataService metadataService; + + protected ScheduledExecutorService scheduledExecutorService; + protected MQClientAPIFactory messagingClientAPIFactory; + protected MQClientAPIFactory operationClientAPIFactory; + protected MQClientAPIFactory transactionClientAPIFactory; + + public ClusterServiceManager(RPCHook rpcHook) { + this.scheduledExecutorService = Executors.newScheduledThreadPool(3); + this.producerManager = new ProducerManager(); + this.consumerManager = new ConsumerManager(new ConsumerIdsChangeListenerImpl()); + + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + this.messagingClientAPIFactory = new MQClientAPIFactory( + "ClusterMQClient_", + proxyConfig.getRocketmqMQClientNum(), + new DoNothingClientRemotingProcessor(null), + rpcHook, + scheduledExecutorService); + this.operationClientAPIFactory = new MQClientAPIFactory( + "TopicRouteServiceClient_", + 1, + new DoNothingClientRemotingProcessor(null), + rpcHook, + this.scheduledExecutorService + ); + this.transactionClientAPIFactory = new MQClientAPIFactory( + "ClusterTransaction_", + 1, + new ProxyClientRemotingProcessor(producerManager), + rpcHook, + scheduledExecutorService); + + this.topicRouteService = new ClusterTopicRouteService(operationClientAPIFactory); + this.messageService = new ClusterMessageService(this.topicRouteService, this.messagingClientAPIFactory); + this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, rpcHook, + this.transactionClientAPIFactory); + this.proxyRelayService = new ClusterProxyRelayService(this.clusterTransactionService); + this.metadataService = new ClusterMetadataService(topicRouteService, operationClientAPIFactory); + + this.init(); + } + + protected void init() { + this.producerManager.appendProducerChangeListener(new ProducerChangeListenerImpl()); + + this.scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + producerManager.scanNotActiveChannel(); + consumerManager.scanNotActiveChannel(); + } catch (Throwable e) { + log.error("Error occurred when scan not active client channels.", e); + } + }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS); + + this.appendShutdown(scheduledExecutorService::shutdown); + this.appendStartAndShutdown(this.messagingClientAPIFactory); + this.appendStartAndShutdown(this.operationClientAPIFactory); + this.appendStartAndShutdown(this.transactionClientAPIFactory); + this.appendStartAndShutdown(this.topicRouteService); + this.appendStartAndShutdown(this.clusterTransactionService); + this.appendStartAndShutdown(this.metadataService); + } + + @Override + public MessageService getMessageService() { + return this.messageService; + } + + @Override + public TopicRouteService getTopicRouteService() { + return topicRouteService; + } + + @Override + public ProducerManager getProducerManager() { + return this.producerManager; + } + + @Override + public ConsumerManager getConsumerManager() { + return this.consumerManager; + } + + @Override + public TransactionService getTransactionService() { + return this.clusterTransactionService; + } + + @Override + public ProxyRelayService getProxyRelayService() { + return this.proxyRelayService; + } + + @Override + public MetadataService getMetadataService() { + return this.metadataService; + } + + protected static class ConsumerIdsChangeListenerImpl implements ConsumerIdsChangeListener { + + @Override + public void handle(ConsumerGroupEvent event, String group, Object... args) { + + } + + @Override + public void shutdown() { + + } + } + + protected class ProducerChangeListenerImpl implements ProducerChangeListener { + @Override + public void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo) { + if (event == ProducerGroupEvent.GROUP_UNREGISTER) { + getTransactionService().unSubscribeAllTransactionTopic(group); + } + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java new file mode 100644 index 0000000000..6afc86c578 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.service.channel.ChannelManager; +import org.apache.rocketmq.proxy.service.message.LocalMessageService; +import org.apache.rocketmq.proxy.service.message.MessageService; +import org.apache.rocketmq.proxy.service.metadata.LocalMetadataService; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.mqclient.DoNothingClientRemotingProcessor; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.relay.LocalProxyRelayService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.LocalTopicRouteService; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.proxy.service.transaction.LocalTransactionService; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.RPCHook; + +public class LocalServiceManager extends AbstractStartAndShutdown implements ServiceManager { + + private final BrokerController brokerController; + private final TopicRouteService topicRouteService; + private final MessageService messageService; + private final TransactionService transactionService; + private final ProxyRelayService proxyRelayService; + private final MetadataService metadataService; + + private final MQClientAPIFactory mqClientAPIFactory; + private final ChannelManager channelManager; + + private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("LocalServiceManagerScheduledThread")); + + public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { + this.brokerController = brokerController; + this.channelManager = new ChannelManager(); + this.messageService = new LocalMessageService(brokerController, channelManager, rpcHook); + this.mqClientAPIFactory = new MQClientAPIFactory( + "TopicRouteServiceClient_", + 1, + new DoNothingClientRemotingProcessor(null), + rpcHook, + scheduledExecutorService + ); + this.topicRouteService = new LocalTopicRouteService(brokerController, mqClientAPIFactory); + this.transactionService = new LocalTransactionService(brokerController.getBrokerConfig()); + this.proxyRelayService = new LocalProxyRelayService(brokerController, this.transactionService); + this.metadataService = new LocalMetadataService(brokerController); + this.init(); + } + + protected void init() { + this.appendStartAndShutdown(this.mqClientAPIFactory); + this.appendStartAndShutdown(this.topicRouteService); + this.appendStartAndShutdown(new LocalServiceManagerStartAndShutdown()); + } + + @Override + public MessageService getMessageService() { + return this.messageService; + } + + @Override + public TopicRouteService getTopicRouteService() { + return this.topicRouteService; + } + + @Override + public ProducerManager getProducerManager() { + return this.brokerController.getProducerManager(); + } + + @Override + public ConsumerManager getConsumerManager() { + return this.brokerController.getConsumerManager(); + } + + @Override + public TransactionService getTransactionService() { + return this.transactionService; + } + + @Override + public ProxyRelayService getProxyRelayService() { + return this.proxyRelayService; + } + + @Override + public MetadataService getMetadataService() { + return this.metadataService; + } + + private class LocalServiceManagerStartAndShutdown implements StartAndShutdown { + @Override + public void start() throws Exception { + LocalServiceManager.this.scheduledExecutorService.scheduleWithFixedDelay(channelManager::scanAndCleanChannels, 5, 5, TimeUnit.MINUTES); + } + + @Override + public void shutdown() throws Exception { + LocalServiceManager.this.scheduledExecutorService.shutdown(); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java new file mode 100644 index 0000000000..563b567152 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManager.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service; + +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.service.message.MessageService; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; + +public interface ServiceManager extends StartAndShutdown { + MessageService getMessageService(); + + TopicRouteService getTopicRouteService(); + + ProducerManager getProducerManager(); + + ConsumerManager getConsumerManager(); + + TransactionService getTransactionService(); + + ProxyRelayService getProxyRelayService(); + + MetadataService getMetadataService(); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java new file mode 100644 index 0000000000..c186752788 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ServiceManagerFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.remoting.RPCHook; + +public class ServiceManagerFactory { + public static ServiceManager createForLocalMode(BrokerController brokerController) { + return createForLocalMode(brokerController, null); + } + + public static ServiceManager createForLocalMode(BrokerController brokerController, RPCHook rpcHook) { + return new LocalServiceManager(brokerController, rpcHook); + } + + public static ServiceManager createForClusterMode() { + return createForClusterMode(null); + } + + public static ServiceManager createForClusterMode(RPCHook rpcHook) { + return new ClusterServiceManager(rpcHook); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/ChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/ChannelManager.java new file mode 100644 index 0000000000..d730d91182 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/ChannelManager.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import com.google.common.base.Strings; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ChannelManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private final ConcurrentMap clientIdChannelMap = new ConcurrentHashMap<>(); + + public SimpleChannel createChannel(ProxyContext context) { + final String clientId = anonymousChannelId(context); + if (Strings.isNullOrEmpty(clientId)) { + log.warn("ClientId is unexpected null or empty"); + return createChannelInner(context); + } + + SimpleChannel channel = clientIdChannelMap.computeIfAbsent(clientId, k -> createChannelInner(context)); + channel.updateLastAccessTime(); + return channel; + } + + public SimpleChannel createInvocationChannel(ProxyContext context) { + final String clientId = anonymousChannelId(InvocationChannel.class.getName(), context); + final String clientHost = context.getRemoteAddress(); + final String localAddress = context.getLocalAddress(); + if (Strings.isNullOrEmpty(clientId)) { + log.warn("ClientId is unexpected null or empty"); + return new InvocationChannel(clientHost, localAddress); + } + + SimpleChannel channel = clientIdChannelMap.computeIfAbsent(clientId, k -> new InvocationChannel(clientHost, localAddress)); + channel.updateLastAccessTime(); + return channel; + } + + private String anonymousChannelId(ProxyContext context) { + final String clientHost = context.getRemoteAddress(); + final String localAddress = context.getLocalAddress(); + return clientHost + "@" + localAddress; + } + + private String anonymousChannelId(String key, ProxyContext context) { + final String clientHost = context.getRemoteAddress(); + final String localAddress = context.getLocalAddress(); + return key + "@" + clientHost + "@" + localAddress; + } + + private SimpleChannel createChannelInner(ProxyContext context) { + return new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress()); + } + + public void scanAndCleanChannels() { + try { + Iterator> iterator = clientIdChannelMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (!entry.getValue().isActive()) { + iterator.remove(); + } else { + entry.getValue().clearExpireContext(); + } + } + } catch (Throwable e) { + log.error("Unexpected exception", e); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationChannel.java new file mode 100644 index 0000000000..00e8cea99c --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationChannel.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import io.netty.channel.ChannelFuture; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class InvocationChannel extends SimpleChannel { + protected final ConcurrentMap inFlightRequestMap; + + public InvocationChannel(String remoteAddress, String localAddress) { + super(remoteAddress, localAddress); + this.inFlightRequestMap = new ConcurrentHashMap<>(); + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + if (msg instanceof RemotingCommand) { + RemotingCommand responseCommand = (RemotingCommand) msg; + InvocationContextInterface context = inFlightRequestMap.remove(responseCommand.getOpaque()); + if (null != context) { + context.handle(responseCommand); + } + inFlightRequestMap.remove(responseCommand.getOpaque()); + } + return super.writeAndFlush(msg); + } + + @Override + public boolean isWritable() { + return inFlightRequestMap.size() > 0; + } + + @Override + public void registerInvocationContext(int opaque, InvocationContextInterface context) { + inFlightRequestMap.put(opaque, context); + } + + @Override + public void eraseInvocationContext(int opaque) { + inFlightRequestMap.remove(opaque); + } + + @Override + public void clearExpireContext() { + Iterator> iterator = inFlightRequestMap.entrySet().iterator(); + int count = 0; + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getValue().expired(ConfigurationManager.getProxyConfig().getChannelExpiredInSeconds())) { + iterator.remove(); + count++; + log.debug("An expired request is found, request: {}", entry.getValue()); + } + } + if (count > 0) { + log.warn("[BUG] {} expired in-flight requests is cleaned.", count); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContext.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContext.java new file mode 100644 index 0000000000..9fb488eb9b --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class InvocationContext implements InvocationContextInterface { + private final CompletableFuture response; + private final long timestamp = System.currentTimeMillis(); + + public InvocationContext(CompletableFuture resp) { + this.response = resp; + } + + public boolean expired(long expiredTimeSec) { + return System.currentTimeMillis() - timestamp >= Duration.ofSeconds(expiredTimeSec).toMillis(); + } + + public CompletableFuture getResponse() { + return response; + } + + public void handle(RemotingCommand remotingCommand) { + response.complete(remotingCommand); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContextInterface.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContextInterface.java new file mode 100644 index 0000000000..0db9516486 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/InvocationContextInterface.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public interface InvocationContextInterface { + void handle(RemotingCommand remotingCommand); + + boolean expired(long expiredTimeSec); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java new file mode 100644 index 0000000000..35e817b5a2 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannel.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import com.google.common.base.Strings; +import io.netty.channel.AbstractChannel; +import io.netty.channel.Channel; +import io.netty.channel.ChannelConfig; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelId; +import io.netty.channel.ChannelMetadata; +import io.netty.channel.ChannelOutboundBuffer; +import io.netty.channel.DefaultChannelPromise; +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.GlobalEventExecutor; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; + +/** + * SimpleChannel is used to handle writeAndFlush situation in processor + * + * @see io.netty.channel.ChannelHandlerContext#writeAndFlush + * @see io.netty.channel.Channel#writeAndFlush + */ +public class SimpleChannel extends AbstractChannel { + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected final String remoteAddress; + protected final String localAddress; + + protected long lastAccessTime; + protected ChannelHandlerContext channelHandlerContext; + + /** + * Creates a new instance. + * + * @param parent the parent of this channel. {@code null} if there's no parent. + * @param remoteAddress Remote address + * @param localAddress Local address + */ + public SimpleChannel(Channel parent, String remoteAddress, String localAddress) { + this(parent, null, remoteAddress, localAddress); + } + + public SimpleChannel(Channel parent, ChannelId id, String remoteAddress, String localAddress) { + super(parent, id); + lastAccessTime = System.currentTimeMillis(); + this.remoteAddress = remoteAddress; + this.localAddress = localAddress; + this.channelHandlerContext = new SimpleChannelHandlerContext(this); + } + + public SimpleChannel(String remoteAddress, String localAddress) { + this(null, remoteAddress, localAddress); + } + + @Override + protected AbstractUnsafe newUnsafe() { + return null; + } + + @Override + protected boolean isCompatible(EventLoop loop) { + return false; + } + + private static SocketAddress parseSocketAddress(String address) { + if (Strings.isNullOrEmpty(address)) { + return null; + } + + String[] segments = address.split(":"); + if (2 == segments.length) { + return new InetSocketAddress(segments[0], Integer.parseInt(segments[1])); + } + + return null; + } + + @Override + protected SocketAddress localAddress0() { + return parseSocketAddress(localAddress); + } + + @Override + public SocketAddress localAddress() { + return localAddress0(); + } + + @Override + public SocketAddress remoteAddress() { + return remoteAddress0(); + } + + @Override + protected SocketAddress remoteAddress0() { + return parseSocketAddress(remoteAddress); + } + + @Override + protected void doBind(SocketAddress localAddress) throws Exception { + + } + + @Override + protected void doDisconnect() throws Exception { + + } + + @Override + public ChannelFuture close() { + DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE); + promise.setSuccess(); + return promise; + } + + @Override + protected void doClose() throws Exception { + + } + + @Override + protected void doBeginRead() throws Exception { + + } + + @Override + protected void doWrite(ChannelOutboundBuffer in) throws Exception { + + } + + @Override + public ChannelConfig config() { + return null; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public boolean isActive() { + return (System.currentTimeMillis() - lastAccessTime) <= 120L * 1000; + } + + @Override + public ChannelMetadata metadata() { + return null; + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE); + promise.setSuccess(); + return promise; + } + + public void updateLastAccessTime() { + this.lastAccessTime = System.currentTimeMillis(); + } + + public void registerInvocationContext(int opaque, InvocationContextInterface context) { + + } + + public void eraseInvocationContext(int opaque) { + + } + + public void clearExpireContext() { + + } + + public ChannelHandlerContext getChannelHandlerContext() { + return channelHandlerContext; + } +} \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannelHandlerContext.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannelHandlerContext.java new file mode 100644 index 0000000000..801c62ee5f --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/channel/SimpleChannelHandlerContext.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.channel; + +import io.netty.buffer.ByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.ChannelProgressivePromise; +import io.netty.channel.ChannelPromise; +import io.netty.util.Attribute; +import io.netty.util.AttributeKey; +import io.netty.util.concurrent.EventExecutor; +import java.net.SocketAddress; +import org.apache.commons.lang3.NotImplementedException; + +public class SimpleChannelHandlerContext implements ChannelHandlerContext { + + private final Channel channel; + + public SimpleChannelHandlerContext(Channel channel) { + this.channel = channel; + } + + @Override + public Channel channel() { + return channel; + } + + @Override + public EventExecutor executor() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public String name() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandler handler() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public boolean isRemoved() { + return false; + } + + @Override + public ChannelHandlerContext fireChannelRegistered() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelUnregistered() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelActive() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelInactive() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireExceptionCaught(Throwable cause) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireUserEventTriggered(Object evt) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelRead(Object msg) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelReadComplete() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext fireChannelWritabilityChanged() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture bind(SocketAddress localAddress) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture disconnect() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture close() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture deregister() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture bind(SocketAddress localAddress, ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture connect(SocketAddress remoteAddress, SocketAddress localAddress, ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture disconnect(ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture close(ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture deregister(ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext read() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture write(Object msg) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture write(Object msg, ChannelPromise promise) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelHandlerContext flush() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture writeAndFlush(Object msg, ChannelPromise promise) { + return channel.writeAndFlush(msg, promise); + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + return channel.writeAndFlush(msg); + } + + @Override + public ChannelPipeline pipeline() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ByteBufAllocator alloc() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelPromise newPromise() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelProgressivePromise newProgressivePromise() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture newSucceededFuture() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelFuture newFailedFuture(Throwable cause) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public ChannelPromise voidPromise() { + throw new NotImplementedException("Not implemented"); + } + + @Override + public Attribute attr(AttributeKey key) { + throw new NotImplementedException("Not implemented"); + } + + @Override + public boolean hasAttr(AttributeKey attributeKey) { + return false; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java new file mode 100644 index 0000000000..e5b7e95330 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.message; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class ClusterMessageService implements MessageService { + private final TopicRouteService topicRouteService; + private final MQClientAPIFactory mqClientAPIFactory; + + public ClusterMessageService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) { + this.topicRouteService = topicRouteService; + this.mqClientAPIFactory = mqClientAPIFactory; + } + + @Override + public CompletableFuture> sendMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + List msgList, SendMessageRequestHeader requestHeader, long timeoutMillis) { + CompletableFuture> future; + if (msgList.size() == 1) { + future = this.mqClientAPIFactory.getClient().sendMessageAsync( + messageQueue.getBrokerAddr(), + messageQueue.getBrokerName(), msgList.get(0), requestHeader, timeoutMillis) + .thenApply(Lists::newArrayList); + } else { + future = this.mqClientAPIFactory.getClient().sendMessageAsync( + messageQueue.getBrokerAddr(), + messageQueue.getBrokerName(), msgList, requestHeader, timeoutMillis) + .thenApply(Lists::newArrayList); + } + return future; + } + + @Override + public CompletableFuture sendMessageBack(ProxyContext ctx, ReceiptHandle handle, String messageId, + ConsumerSendMsgBackRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().sendMessageBackAsync( + this.resolveBrokerAddr(handle), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture endTransactionOneway(ProxyContext ctx, String brokerName, EndTransactionRequestHeader requestHeader, + long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.mqClientAPIFactory.getClient().endTransactionOneway( + this.resolveBrokerAddr(brokerName), + requestHeader, + "end transaction from proxy", + timeoutMillis + ); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + @Override + public CompletableFuture popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PopMessageRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().popMessageAsync( + messageQueue.getBrokerAddr(), + messageQueue.getBrokerName(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, + ChangeInvisibleTimeRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().changeInvisibleTimeAsync( + this.resolveBrokerAddr(handle), + handle.getBrokerName(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId, + AckMessageRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().ackMessageAsync( + this.resolveBrokerAddr(handle), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().pullMessageAsync( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture queryConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().queryConsumerOffsetWithFuture( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture updateConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().updateConsumerOffsetOneWay( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, + LockBatchRequestBody requestBody, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().lockBatchMQWithFuture( + messageQueue.getBrokerAddr(), + requestBody, + timeoutMillis + ); + } + + @Override + public CompletableFuture unlockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, + UnlockBatchRequestBody requestBody, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().unlockBatchMQOneway( + messageQueue.getBrokerAddr(), + requestBody, + timeoutMillis + ); + } + + @Override + public CompletableFuture getMaxOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + GetMaxOffsetRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().getMaxOffset( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + + @Override + public CompletableFuture getMinOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + GetMinOffsetRequestHeader requestHeader, long timeoutMillis) { + return this.mqClientAPIFactory.getClient().getMinOffset( + messageQueue.getBrokerAddr(), + requestHeader, + timeoutMillis + ); + } + + protected String resolveBrokerAddr(ReceiptHandle handle) { + return resolveBrokerAddr(handle.getBrokerName()); + } + + protected String resolveBrokerAddr(String brokerName) { + try { + return this.topicRouteService.getBrokerAddr(brokerName); + } catch (Throwable t) { + throw new ProxyException(ProxyExceptionCode.INVALID_BROKER_NAME, "broker " + brokerName + " cannot find", t); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java new file mode 100644 index 0000000000..93878f286c --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.message; + +import io.netty.channel.ChannelHandlerContext; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.service.channel.ChannelManager; +import org.apache.rocketmq.proxy.service.channel.InvocationContext; +import org.apache.rocketmq.proxy.service.channel.SimpleChannel; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LocalMessageService implements MessageService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private final BrokerController brokerController; + private final ChannelManager channelManager; + + public LocalMessageService(BrokerController brokerController, ChannelManager channelManager, RPCHook rpcHook) { + this.brokerController = brokerController; + this.channelManager = channelManager; + } + + @Override + public CompletableFuture> sendMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + List msgList, SendMessageRequestHeader requestHeader, long timeoutMillis) { + byte[] body; + String messageId; + if (msgList.size() > 1) { + requestHeader.setBatch(true); + MessageBatch msgBatch = MessageBatch.generateFromList(msgList); + MessageClientIDSetter.setUniqID(msgBatch); + body = msgBatch.encode(); + msgBatch.setBody(body); + messageId = MessageClientIDSetter.getUniqID(msgBatch); + } else { + Message message = msgList.get(0); + body = message.getBody(); + messageId = MessageClientIDSetter.getUniqID(message); + } + RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + request.setBody(body); + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createInvocationChannel(ctx); + InvocationContext invocationContext = new InvocationContext(future); + channel.registerInvocationContext(request.getOpaque(), invocationContext); + ChannelHandlerContext simpleChannelHandlerContext = channel.getChannelHandlerContext(); + try { + RemotingCommand response = brokerController.getSendMessageProcessor().processRequest(simpleChannelHandlerContext, request); + if (response != null) { + invocationContext.handle(response); + channel.eraseInvocationContext(request.getOpaque()); + } + } catch (Exception e) { + future.completeExceptionally(e); + channel.eraseInvocationContext(request.getOpaque()); + log.error("Failed to process sendMessage command", e); + } + return future.thenApply(r -> { + SendResult sendResult = new SendResult(); + SendMessageResponseHeader responseHeader = (SendMessageResponseHeader) r.readCustomHeader(); + SendStatus sendStatus; + switch (r.getCode()) { + case ResponseCode.FLUSH_DISK_TIMEOUT: { + sendStatus = SendStatus.FLUSH_DISK_TIMEOUT; + break; + } + case ResponseCode.FLUSH_SLAVE_TIMEOUT: { + sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT; + break; + } + case ResponseCode.SLAVE_NOT_AVAILABLE: { + sendStatus = SendStatus.SLAVE_NOT_AVAILABLE; + break; + } + case ResponseCode.SUCCESS: { + sendStatus = SendStatus.SEND_OK; + break; + } + default: { + throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark()); + } + } + sendResult.setSendStatus(sendStatus); + sendResult.setMsgId(messageId); + sendResult.setMessageQueue(new MessageQueue(requestHeader.getTopic(), brokerController.getBrokerConfig().getBrokerName(), requestHeader.getQueueId())); + sendResult.setQueueOffset(responseHeader.getQueueOffset()); + sendResult.setTransactionId(responseHeader.getTransactionId()); + sendResult.setOffsetMsgId(responseHeader.getMsgId()); + return Collections.singletonList(sendResult); + }); + } + + @Override + public CompletableFuture sendMessageBack(ProxyContext ctx, ReceiptHandle handle, String messageId, + ConsumerSendMsgBackRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getSendMessageProcessor() + .processRequest(channelHandlerContext, command); + future.complete(response); + } catch (Exception e) { + log.error("Fail to process sendMessageBack command", e); + future.completeExceptionally(e); + } + return future; + } + + @Override + public CompletableFuture endTransactionOneway(ProxyContext ctx, String brokerName, EndTransactionRequestHeader requestHeader, + long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader); + try { + brokerController.getEndTransactionProcessor() + .processRequest(channelHandlerContext, command); + future.complete(null); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + + @Override + public CompletableFuture popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PopMessageRequestHeader requestHeader, long timeoutMillis) { + requestHeader.setBornTime(System.currentTimeMillis()); + RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createInvocationChannel(ctx); + InvocationContext invocationContext = new InvocationContext(future); + channel.registerInvocationContext(request.getOpaque(), invocationContext); + ChannelHandlerContext simpleChannelHandlerContext = channel.getChannelHandlerContext(); + try { + RemotingCommand response = brokerController.getPopMessageProcessor().processRequest(simpleChannelHandlerContext, request); + if (response != null) { + invocationContext.handle(response); + channel.eraseInvocationContext(request.getOpaque()); + } + } catch (Exception e) { + future.completeExceptionally(e); + channel.eraseInvocationContext(request.getOpaque()); + log.error("Failed to process popMessage command", e); + } + return future.thenApply(r -> { + PopStatus popStatus; + List messageExtList = new ArrayList<>(); + switch (r.getCode()) { + case ResponseCode.SUCCESS: + popStatus = PopStatus.FOUND; + ByteBuffer byteBuffer = ByteBuffer.wrap(r.getBody()); + messageExtList = MessageDecoder.decodesBatch( + byteBuffer, + true, + false, + true + ); + break; + case ResponseCode.POLLING_FULL: + popStatus = PopStatus.POLLING_FULL; + break; + case ResponseCode.POLLING_TIMEOUT: + case ResponseCode.PULL_NOT_FOUND: + popStatus = PopStatus.POLLING_NOT_FOUND; + break; + default: + throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, r.getRemark()); + } + PopResult popResult = new PopResult(popStatus, messageExtList); + PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) r.readCustomHeader(); + + if (popStatus == PopStatus.FOUND) { + Map startOffsetInfo; + Map> msgOffsetInfo; + Map orderCountInfo; + popResult.setInvisibleTime(responseHeader.getInvisibleTime()); + popResult.setPopTime(responseHeader.getPopTime()); + startOffsetInfo = ExtraInfoUtil.parseStartOffsetInfo(responseHeader.getStartOffsetInfo()); + msgOffsetInfo = ExtraInfoUtil.parseMsgOffsetInfo(responseHeader.getMsgOffsetInfo()); + orderCountInfo = ExtraInfoUtil.parseOrderCountInfo(responseHeader.getOrderCountInfo()); + // + Map> sortMap = new HashMap<>(16); + for (MessageExt messageExt : messageExtList) { + String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); + if (!sortMap.containsKey(key)) { + sortMap.put(key, new ArrayList<>(4)); + } + sortMap.get(key).add(messageExt.getQueueOffset()); + } + Map map = new HashMap<>(5); + for (MessageExt messageExt : messageExtList) { + if (startOffsetInfo == null) { + // we should set the check point info to extraInfo field , if the command is popMsg + // find pop ck offset + String key = messageExt.getTopic() + messageExt.getQueueId(); + if (!map.containsKey(messageExt.getTopic() + messageExt.getQueueId())) { + map.put(key, ExtraInfoUtil.buildExtraInfo(messageExt.getQueueOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), responseHeader.getReviveQid(), + messageExt.getTopic(), messageQueue.getBrokerName(), messageExt.getQueueId())); + } + messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK, map.get(key) + MessageConst.KEY_SEPARATOR + messageExt.getQueueOffset()); + } else { + String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); + int index = sortMap.get(key).indexOf(messageExt.getQueueOffset()); + Long msgQueueOffset = msgOffsetInfo.get(key).get(index); + if (msgQueueOffset != messageExt.getQueueOffset()) { + log.warn("Queue offset [{}] of msg is strange, not equal to the stored in msg, {}", msgQueueOffset, messageExt); + } + + messageExt.getProperties().put(MessageConst.PROPERTY_POP_CK, + ExtraInfoUtil.buildExtraInfo(startOffsetInfo.get(key), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), + responseHeader.getReviveQid(), messageExt.getTopic(), messageQueue.getBrokerName(), messageExt.getQueueId(), msgQueueOffset) + ); + if (requestHeader.isOrder() && orderCountInfo != null) { + Integer count = orderCountInfo.get(key); + if (count != null && count > 0) { + messageExt.setReconsumeTimes(count); + } + } + } + messageExt.getProperties().computeIfAbsent(MessageConst.PROPERTY_FIRST_POP_TIME, k -> String.valueOf(responseHeader.getPopTime())); + messageExt.setBrokerName(messageExt.getBrokerName()); + messageExt.setTopic(messageQueue.getTopic()); + } + } + return popResult; + }); + } + + @Override + public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, + ChangeInvisibleTimeRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getChangeInvisibleTimeProcessor() + .processRequest(channelHandlerContext, command); + future.complete(response); + } catch (Exception e) { + log.error("Fail to process changeInvisibleTime command", e); + future.completeExceptionally(e); + } + return future.thenApply(r -> { + ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) r.readCustomHeader(); + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == r.getCode()) { + ackResult.setStatus(AckStatus.OK); + } else { + ackResult.setStatus(AckStatus.NO_EXIST); + } + ackResult.setPopTime(responseHeader.getPopTime()); + ackResult.setExtraInfo(ReceiptHandle.builder() + .startOffset(handle.getStartOffset()) + .retrieveTime(responseHeader.getPopTime()) + .invisibleTime(responseHeader.getInvisibleTime()) + .reviveQueueId(responseHeader.getReviveQid()) + .topicType(handle.getTopicType()) + .brokerName(handle.getBrokerName()) + .queueId(handle.getQueueId()) + .offset(handle.getOffset()) + .build() + .encode()); + return ackResult; + }); + } + + @Override + public CompletableFuture ackMessage(ProxyContext ctx, ReceiptHandle handle, String messageId, + AckMessageRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); + RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getAckMessageProcessor() + .processRequest(channelHandlerContext, command); + future.complete(response); + } catch (Exception e) { + log.error("Fail to process ackMessage command", e); + future.completeExceptionally(e); + } + return future.thenApply(r -> { + AckResult ackResult = new AckResult(); + if (ResponseCode.SUCCESS == r.getCode()) { + ackResult.setStatus(AckStatus.OK); + } else { + ackResult.setStatus(AckStatus.NO_EXIST); + } + return ackResult; + }); + } + + @Override + public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("pullMessage is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture queryConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + QueryConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("queryConsumerOffset is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture updateConsumerOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("updateConsumerOffset is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture> lockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, + LockBatchRequestBody requestBody, long timeoutMillis) { + throw new NotImplementedException("lockBatchMQ is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture unlockBatchMQ(ProxyContext ctx, AddressableMessageQueue messageQueue, + UnlockBatchRequestBody requestBody, long timeoutMillis) { + throw new NotImplementedException("unlockBatchMQ is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture getMaxOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + GetMaxOffsetRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("getMaxOffset is not implemented in LocalMessageService"); + } + + @Override + public CompletableFuture getMinOffset(ProxyContext ctx, AddressableMessageQueue messageQueue, + GetMinOffsetRequestHeader requestHeader, long timeoutMillis) { + throw new NotImplementedException("getMinOffset is not implemented in LocalMessageService"); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java new file mode 100644 index 0000000000..73048dbbc2 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.message; + +import java.util.HashMap; +import org.apache.rocketmq.remoting.CommandCustomHeader; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class LocalRemotingCommand extends RemotingCommand { + + public static LocalRemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader) { + LocalRemotingCommand cmd = new LocalRemotingCommand(); + cmd.setCode(code); + cmd.writeCustomHeader(customHeader); + cmd.setExtFields(new HashMap<>()); + setCmdVersion(cmd); + return cmd; + } + + @Override + public CommandCustomHeader decodeCommandCustomHeader( + Class classHeader) throws RemotingCommandException { + return classHeader.cast(readCustomHeader()); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java new file mode 100644 index 0000000000..ee35fa03f2 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.message; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public interface MessageService { + + CompletableFuture> sendMessage( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + List msgList, + SendMessageRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture sendMessageBack( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + ConsumerSendMsgBackRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture endTransactionOneway( + ProxyContext ctx, + String brokerName, + EndTransactionRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture popMessage( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + PopMessageRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture changeInvisibleTime( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + ChangeInvisibleTimeRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture ackMessage( + ProxyContext ctx, + ReceiptHandle handle, + String messageId, + AckMessageRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture pullMessage( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + PullMessageRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture queryConsumerOffset( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + QueryConsumerOffsetRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture updateConsumerOffset( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + UpdateConsumerOffsetRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture> lockBatchMQ( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + LockBatchRequestBody requestBody, + long timeoutMillis + ); + + CompletableFuture unlockBatchMQ( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + UnlockBatchRequestBody requestBody, + long timeoutMillis + ); + + CompletableFuture getMaxOffset( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + GetMaxOffsetRequestHeader requestHeader, + long timeoutMillis + ); + + CompletableFuture getMinOffset( + ProxyContext ctx, + AddressableMessageQueue messageQueue, + GetMinOffsetRequestHeader requestHeader, + long timeoutMillis + ); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java new file mode 100644 index 0000000000..0291289340 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.metadata; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.LoadingCache; +import java.util.Optional; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.AbstractCacheLoader; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.route.TopicRouteHelper; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; + +public class ClusterMetadataService extends AbstractStartAndShutdown implements MetadataService { + protected static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + private static final long DEFAULT_TIMEOUT = 3000; + + private final TopicRouteService topicRouteService; + private final MQClientAPIFactory mqClientAPIFactory; + + protected final ThreadPoolExecutor cacheRefreshExecutor; + + protected final LoadingCache topicConfigCache; + protected final static TopicConfigAndQueueMapping EMPTY_TOPIC_CONFIG = new TopicConfigAndQueueMapping(); + + protected final LoadingCache subscriptionGroupConfigCache; + protected final static SubscriptionGroupConfig EMPTY_SUBSCRIPTION_GROUP_CONFIG = new SubscriptionGroupConfig(); + + public ClusterMetadataService(TopicRouteService topicRouteService, MQClientAPIFactory mqClientAPIFactory) { + this.topicRouteService = topicRouteService; + this.mqClientAPIFactory = mqClientAPIFactory; + + ProxyConfig config = ConfigurationManager.getProxyConfig(); + this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( + config.getMetadataThreadPoolNums(), + config.getMetadataThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + "MetadataCacheRefresh", + config.getMetadataThreadPoolQueueCapacity() + ); + this.topicConfigCache = CacheBuilder.newBuilder() + .maximumSize(config.getTopicConfigCacheMaxNum()) + .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) + .build(new ClusterTopicConfigCacheLoader()); + this.subscriptionGroupConfigCache = CacheBuilder.newBuilder() + .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum()) + .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) + .build(new ClusterSubscriptionGroupConfigCacheLoader()); + + this.init(); + } + + protected void init() { + this.appendShutdown(this.cacheRefreshExecutor::shutdown); + } + + @Override + public TopicMessageType getTopicMessageType(String topic) { + TopicConfigAndQueueMapping topicConfigAndQueueMapping; + try { + topicConfigAndQueueMapping = topicConfigCache.get(topic); + } catch (Exception e) { + return TopicMessageType.UNSPECIFIED; + } + if (topicConfigAndQueueMapping.equals(EMPTY_TOPIC_CONFIG)) { + return TopicMessageType.UNSPECIFIED; + } + return topicConfigAndQueueMapping.getTopicMessageType(); + } + + @Override + public SubscriptionGroupConfig getSubscriptionGroupConfig(String group) { + SubscriptionGroupConfig config; + try { + config = this.subscriptionGroupConfigCache.get(group); + } catch (Exception e) { + return null; + } + if (config == EMPTY_SUBSCRIPTION_GROUP_CONFIG) { + return null; + } + return config; + } + + protected class ClusterSubscriptionGroupConfigCacheLoader extends AbstractCacheLoader { + + public ClusterSubscriptionGroupConfigCacheLoader() { + super(cacheRefreshExecutor); + } + + @Override + protected SubscriptionGroupConfig getDirectly(String consumerGroup) throws Exception { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + String clusterName = config.getRocketMQClusterName(); + Optional brokerDataOptional = findOneBroker(clusterName); + if (brokerDataOptional.isPresent()) { + String brokerAddress = brokerDataOptional.get().selectBrokerAddr(); + return mqClientAPIFactory.getClient().getSubscriptionGroupConfig(brokerAddress, consumerGroup, DEFAULT_TIMEOUT); + } + return EMPTY_SUBSCRIPTION_GROUP_CONFIG; + } + + @Override + protected void onErr(String consumerGroup, Exception e) { + log.error("load subscription config failed. consumerGroup:{}", consumerGroup, e); + } + } + + protected class ClusterTopicConfigCacheLoader extends AbstractCacheLoader { + + public ClusterTopicConfigCacheLoader() { + super(cacheRefreshExecutor); + } + + @Override + protected TopicConfigAndQueueMapping getDirectly(String topic) throws Exception { + Optional brokerDataOptional = findOneBroker(topic); + if (brokerDataOptional.isPresent()) { + String brokerAddress = brokerDataOptional.get().selectBrokerAddr(); + return mqClientAPIFactory.getClient().getTopicConfig(brokerAddress, topic, DEFAULT_TIMEOUT); + } + return EMPTY_TOPIC_CONFIG; + } + + @Override + protected void onErr(String key, Exception e) { + log.error("load topic config failed. topic:{}", key, e); + } + } + + protected Optional findOneBroker(String topic) throws Exception { + try { + return topicRouteService.getAllMessageQueueView(topic).getTopicRouteData().getBrokerDatas().stream().findAny(); + } catch (Exception e) { + if (TopicRouteHelper.isTopicNotExistError(e)) { + return Optional.empty(); + } + throw e; + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java new file mode 100644 index 0000000000..6f06f84888 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/LocalMetadataService.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.metadata; + +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; + +public class LocalMetadataService implements MetadataService { + private final BrokerController brokerController; + + public LocalMetadataService(BrokerController brokerController) { + this.brokerController = brokerController; + } + + @Override + public TopicMessageType getTopicMessageType(String topic) { + TopicConfig topicConfig = brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (topicConfig == null) { + return TopicMessageType.UNSPECIFIED; + } + return topicConfig.getTopicMessageType(); + } + + @Override + public SubscriptionGroupConfig getSubscriptionGroupConfig(String group) { + return this.brokerController.getSubscriptionGroupManager().getSubscriptionGroupTable().get(group); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java new file mode 100644 index 0000000000..6951845e52 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/MetadataService.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.metadata; + +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; + +public interface MetadataService { + + TopicMessageType getTopicMessageType(String topic); + + SubscriptionGroupConfig getSubscriptionGroupConfig(String group); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/DoNothingClientRemotingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/DoNothingClientRemotingProcessor.java new file mode 100644 index 0000000000..5d2be52ab3 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/DoNothingClientRemotingProcessor.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.mqclient; + +import io.netty.channel.ChannelHandlerContext; +import org.apache.rocketmq.client.impl.ClientRemotingProcessor; +import org.apache.rocketmq.client.impl.factory.MQClientInstance; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class DoNothingClientRemotingProcessor extends ClientRemotingProcessor { + + public DoNothingClientRemotingProcessor(MQClientInstance mqClientFactory) { + super(mqClientFactory); + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + return null; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExt.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExt.java new file mode 100644 index 0000000000..c0dddab42a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExt.java @@ -0,0 +1,620 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.mqclient; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.AckCallback; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopCallback; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.client.exception.OffsetNotFoundException; +import org.apache.rocketmq.client.impl.ClientRemotingProcessor; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.LockBatchRequestBody; +import org.apache.rocketmq.common.protocol.body.LockBatchResponseBody; +import org.apache.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class MQClientAPIExt extends MQClientAPIImpl { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final ClientConfig clientConfig; + + public MQClientAPIExt( + ClientConfig clientConfig, + NettyClientConfig nettyClientConfig, + ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook + ) { + super(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); + this.clientConfig = clientConfig; + } + + public boolean updateNameServerAddressList() { + if (this.clientConfig.getNamesrvAddr() != null) { + this.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); + log.info("user specified name server address: {}", this.clientConfig.getNamesrvAddr()); + return true; + } + return false; + } + + protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { + MQClientException ex; + if (!responseFuture.isSendRequestOK()) { + ex = new MQClientException("send request failed", responseFuture.getCause()); + } else if (responseFuture.isTimeout()) { + ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", + responseFuture.getCause()); + } else { + ex = new MQClientException("unknown reason", responseFuture.getCause()); + } + return ex; + } + + public CompletableFuture sendHeartbeatOneway( + String brokerAddr, + HeartbeatData heartbeatData, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + request.setLanguage(clientConfig.getLanguage()); + request.setBody(heartbeatData.encode()); + this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture sendHeartbeatAsync( + String brokerAddr, + HeartbeatData heartbeatData, + long timeoutMillis + ) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + request.setLanguage(clientConfig.getLanguage()); + request.setBody(heartbeatData.encode()); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + if (ResponseCode.SUCCESS == response.getCode()) { + future.complete(response.getVersion()); + } else { + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); + } + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture sendMessageAsync( + String brokerAddr, + String brokerName, + Message msg, + SendMessageRequestHeader requestHeader, + long timeoutMillis + ) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + request.setBody(msg.getBody()); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + try { + future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); + } catch (Exception e) { + future.completeExceptionally(e); + } + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture sendMessageAsync( + String brokerAddr, + String brokerName, + List msgList, + SendMessageRequestHeader requestHeader, + long timeoutMillis + ) { + SendMessageRequestHeaderV2 requestHeaderV2 = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_BATCH_MESSAGE, requestHeaderV2); + + CompletableFuture future = new CompletableFuture<>(); + try { + requestHeader.setBatch(true); + MessageBatch msgBatch = MessageBatch.generateFromList(msgList); + MessageClientIDSetter.setUniqID(msgBatch); + byte[] body = msgBatch.encode(); + msgBatch.setBody(body); + + request.setBody(body); + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + try { + future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); + } catch (Exception e) { + future.completeExceptionally(e); + } + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture sendMessageBackAsync( + String brokerAddr, + ConsumerSendMsgBackRequestHeader requestHeader, + long timeoutMillis + ) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + future.complete(response); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture popMessageAsync( + String brokerAddr, + String brokerName, + PopMessageRequestHeader requestHeader, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.popMessageAsync(brokerName, brokerAddr, requestHeader, timeoutMillis, new PopCallback() { + @Override + public void onSuccess(PopResult popResult) { + future.complete(popResult); + } + + @Override + public void onException(Throwable t) { + future.completeExceptionally(t); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture ackMessageAsync( + String brokerAddr, + AckMessageRequestHeader requestHeader, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.ackMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { + @Override + public void onSuccess(AckResult ackResult) { + future.complete(ackResult); + } + + @Override + public void onException(Throwable t) { + future.completeExceptionally(t); + } + }, requestHeader); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture changeInvisibleTimeAsync( + String brokerAddr, + String brokerName, + ChangeInvisibleTimeRequestHeader requestHeader, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.changeInvisibleTimeAsync(brokerName, brokerAddr, requestHeader, timeoutMillis, + new AckCallback() { + @Override + public void onSuccess(AckResult ackResult) { + future.complete(ackResult); + } + + @Override + public void onException(Throwable t) { + future.completeExceptionally(t); + } + } + ); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture pullMessageAsync( + String brokerAddr, + PullMessageRequestHeader requestHeader, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.pullMessage(brokerAddr, requestHeader, timeoutMillis, CommunicationMode.ASYNC, + new PullCallback() { + @Override + public void onSuccess(PullResult pullResult) { + if (pullResult instanceof PullResultExt) { + PullResultExt pullResultExt = (PullResultExt) pullResult; + if (PullStatus.FOUND.equals(pullResult.getPullStatus())) { + List messageExtList = MessageDecoder.decodesBatch( + ByteBuffer.wrap(pullResultExt.getMessageBinary()), + true, + false, + true + ); + pullResult.setMsgFoundList(messageExtList); + } + } + future.complete(pullResult); + } + + @Override + public void onException(Throwable t) { + future.completeExceptionally(t); + } + } + ); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture queryConsumerOffsetWithFuture( + String brokerAddr, + QueryConsumerOffsetRequestHeader requestHeader, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + try { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + future.complete(responseHeader.getOffset()); + } catch (RemotingCommandException e) { + future.completeExceptionally(e); + } + break; + } + case ResponseCode.QUERY_NOT_FOUND: { + future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); + break; + } + default: + break; + } + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture updateConsumerOffsetOneWay( + String brokerAddr, + UpdateConsumerOffsetRequestHeader header, + long timeoutMillis + ) { + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, header); + this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis); + future.complete(null); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture> getConsumerListByGroupAsync( + String brokerAddr, + GetConsumerListByGroupRequestHeader requestHeader, + long timeoutMillis + ) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader); + + CompletableFuture> future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerListByGroupResponseBody body = + GetConsumerListByGroupResponseBody.decode(response.getBody(), GetConsumerListByGroupResponseBody.class); + future.complete(body.getConsumerIdList()); + return; + } + } + /* + @see org.apache.rocketmq.broker.processor.ConsumerManageProcessor#getConsumerListByGroup, + * broker will return {@link ResponseCode.SYSTEM_ERROR} if there is no consumer. + */ + case ResponseCode.SYSTEM_ERROR: { + future.complete(Collections.emptyList()); + return; + } + default: + break; + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture getMaxOffset(String brokerAddr, GetMaxOffsetRequestHeader requestHeader, + long timeoutMillis) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + if (ResponseCode.SUCCESS == response.getCode()) { + try { + GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); + future.complete(responseHeader.getOffset()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture getMinOffset(String brokerAddr, GetMinOffsetRequestHeader requestHeader, + long timeoutMillis) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + if (ResponseCode.SUCCESS == response.getCode()) { + try { + GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); + future.complete(responseHeader.getOffset()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture searchOffset(String brokerAddr, SearchOffsetRequestHeader requestHeader, + long timeoutMillis) { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); + + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + if (response.getCode() == ResponseCode.SUCCESS) { + try { + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + future.complete(responseHeader.getOffset()); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + return future; + } + + public CompletableFuture> lockBatchMQWithFuture(String brokerAddr, + LockBatchRequestBody requestBody, long timeoutMillis) { + CompletableFuture> future = new CompletableFuture<>(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); + request.setBody(requestBody.encode()); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + if (response.getCode() == ResponseCode.SUCCESS) { + try { + LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); + Set messageQueues = responseBody.getLockOKMQSet(); + future.complete(messageQueues); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } + future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + + public CompletableFuture unlockBatchMQOneway(String brokerAddr, + UnlockBatchRequestBody requestBody, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null); + request.setBody(requestBody.encode()); + try { + this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis); + future.complete(null); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + + public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + future.complete(response); + } else { + future.completeExceptionally(processNullResponseErr(responseFuture)); + } + }); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } + + public CompletableFuture invokeOneway(String brokerAddr, RemotingCommand request, long timeoutMillis) { + CompletableFuture future = new CompletableFuture<>(); + try { + this.getRemotingClient().invokeOneway(brokerAddr, request, timeoutMillis); + future.complete(null); + } catch (Exception e) { + future.completeExceptionally(e); + } + return future; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIFactory.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIFactory.java new file mode 100644 index 0000000000..0b813ae608 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIFactory.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.mqclient; + +import java.time.Duration; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.impl.ClientRemotingProcessor; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.remoting.RPCHook; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; + +public class MQClientAPIFactory implements StartAndShutdown { + + private MQClientAPIExt[] clients; + private final String namePrefix; + private final int clientNum; + private final ClientRemotingProcessor clientRemotingProcessor; + private final RPCHook rpcHook; + private final ScheduledExecutorService scheduledExecutorService; + + public MQClientAPIFactory(String namePrefix, int clientNum, + ClientRemotingProcessor clientRemotingProcessor, + RPCHook rpcHook, ScheduledExecutorService scheduledExecutorService) { + this.namePrefix = namePrefix; + this.clientNum = clientNum; + this.clientRemotingProcessor = clientRemotingProcessor; + this.rpcHook = rpcHook; + this.scheduledExecutorService = scheduledExecutorService; + + this.init(); + } + + protected void init() { + System.setProperty(ClientConfig.SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "false"); + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + if (StringUtils.isEmpty(proxyConfig.getNameSrvDomain())) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, proxyConfig.getNameSrvAddr()); + } else { + System.setProperty("rocketmq.namesrv.domain", proxyConfig.getNameSrvDomain()); + System.setProperty("rocketmq.namesrv.domain.subgroup", proxyConfig.getNameSrvDomainSubgroup()); + } + } + + public MQClientAPIExt getClient() { + if (clients.length == 1) { + return this.clients[0]; + } + int index = ThreadLocalRandom.current().nextInt(this.clients.length); + return this.clients[index]; + } + + @Override + public void start() throws Exception { + this.clients = new MQClientAPIExt[this.clientNum]; + + for (int i = 0; i < this.clientNum; i++) { + clients[i] = createAndStart(this.namePrefix + "N_" + i); + } + } + + @Override + public void shutdown() throws Exception { + for (int i = 0; i < this.clientNum; i++) { + clients[i].shutdown(); + } + } + + protected MQClientAPIExt createAndStart(String instanceName) { + ClientConfig clientConfig = new ClientConfig(); + clientConfig.setInstanceName(instanceName); + clientConfig.setDecodeReadBody(true); + clientConfig.setDecodeDecompressBody(false); + + NettyClientConfig nettyClientConfig = new NettyClientConfig(); + nettyClientConfig.setDisableCallbackExecutor(true); + + MQClientAPIExt mqClientAPIExt = new MQClientAPIExt(clientConfig, nettyClientConfig, + clientRemotingProcessor, + rpcHook); + + if (!mqClientAPIExt.updateNameServerAddressList()) { + this.scheduledExecutorService.scheduleAtFixedRate( + mqClientAPIExt::fetchNameServerAddr, + Duration.ofSeconds(10).toMillis(), + Duration.ofMinutes(2).toMillis(), + TimeUnit.MILLISECONDS + ); + } + mqClientAPIExt.start(); + return mqClientAPIExt; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessor.java new file mode 100644 index 0000000000..5cb36d1a28 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/mqclient/ProxyClientRemotingProcessor.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.mqclient; + +import io.netty.channel.ChannelHandlerContext; +import java.nio.ByteBuffer; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.client.impl.ClientRemotingProcessor; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class ProxyClientRemotingProcessor extends ClientRemotingProcessor { + private final ProducerManager producerManager; + + public ProxyClientRemotingProcessor(ProducerManager producerManager) { + super(null); + this.producerManager = producerManager; + } + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + if (request.getCode() == RequestCode.CHECK_TRANSACTION_STATE) { + return this.checkTransactionState(ctx, request); + } + return null; + } + + @Override + public RemotingCommand checkTransactionState(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); + final MessageExt messageExt = MessageDecoder.decode(byteBuffer, true, false, false); + if (messageExt != null) { + final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + if (group != null) { + CheckTransactionStateRequestHeader requestHeader = + (CheckTransactionStateRequestHeader) request.decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class); + request.writeCustomHeader(requestHeader); + request.addExtField(ProxyUtils.BROKER_ADDR, RemotingUtil.socketAddress2String(ctx.channel().remoteAddress())); + this.producerManager.getAvailableChannel(group).writeAndFlush(request); + } + } + return null; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java new file mode 100644 index 0000000000..27a9c36edf --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/AbstractProxyRelayService.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public abstract class AbstractProxyRelayService implements ProxyRelayService { + + protected final TransactionService transactionService; + + public AbstractProxyRelayService(TransactionService transactionService) { + this.transactionService = transactionService; + } + + @Override + public RelayData processCheckTransactionState(ProxyContext context, + RemotingCommand command, CheckTransactionStateRequestHeader header, MessageExt messageExt) { + CompletableFuture> future = new CompletableFuture<>(); + String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + TransactionData transactionData = transactionService.addTransactionDataByBrokerAddr( + command.getExtFields().get(ProxyUtils.BROKER_ADDR), + group, + header.getTranStateTableOffset(), + header.getCommitLogOffset(), + header.getTransactionId(), + messageExt); + if (transactionData == null) { + throw new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, + String.format("add transaction data failed. request:%s, message:%s", command, messageExt)); + } + future.exceptionally(throwable -> { + this.transactionService.onSendCheckTransactionStateFailed(context, group, transactionData); + return null; + }); + return new RelayData<>(transactionData, future); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java new file mode 100644 index 0000000000..65ffeeb657 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ClusterProxyRelayService.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +/** + * not implement yet + */ +public class ClusterProxyRelayService extends AbstractProxyRelayService { + + public ClusterProxyRelayService(TransactionService transactionService) { + super(transactionService); + } + + @Override + public CompletableFuture> processGetConsumerRunningInfo( + ProxyContext context, RemotingCommand command, + GetConsumerRunningInfoRequestHeader header) { + return null; + } + + @Override + public CompletableFuture> processConsumeMessageDirectly( + ProxyContext context, RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header) { + return null; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayService.java new file mode 100644 index 0000000000..35aa5d073c --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayService.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.channel.SimpleChannel; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.RemotingServer; +import org.apache.rocketmq.remoting.netty.NettyRemotingAbstract; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public class LocalProxyRelayService extends AbstractProxyRelayService { + + private final BrokerController brokerController; + + public LocalProxyRelayService(BrokerController brokerController, TransactionService transactionService) { + super(transactionService); + this.brokerController = brokerController; + } + + @Override + public CompletableFuture> processGetConsumerRunningInfo( + ProxyContext context, RemotingCommand command, GetConsumerRunningInfoRequestHeader header) { + CompletableFuture> future = new CompletableFuture<>(); + future.thenAccept(proxyOutResult -> { + RemotingServer remotingServer = this.brokerController.getRemotingServer(); + if (remotingServer instanceof NettyRemotingAbstract) { + NettyRemotingAbstract nettyRemotingAbstract = (NettyRemotingAbstract) remotingServer; + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(null); + remotingCommand.setOpaque(command.getOpaque()); + remotingCommand.setCode(proxyOutResult.getCode()); + remotingCommand.setRemark(proxyOutResult.getRemark()); + if (proxyOutResult.getCode() == ResponseCode.SUCCESS && proxyOutResult.getResult() != null) { + ConsumerRunningInfo consumerRunningInfo = proxyOutResult.getResult(); + remotingCommand.setBody(consumerRunningInfo.encode()); + } + SimpleChannel simpleChannel = new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress()); + nettyRemotingAbstract.processResponseCommand(simpleChannel.getChannelHandlerContext(), remotingCommand); + } + }); + return future; + } + + @Override + public CompletableFuture> processConsumeMessageDirectly( + ProxyContext context, RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header) { + CompletableFuture> future = new CompletableFuture<>(); + future.thenAccept(proxyOutResult -> { + RemotingServer remotingServer = this.brokerController.getRemotingServer(); + if (remotingServer instanceof NettyRemotingAbstract) { + NettyRemotingAbstract nettyRemotingAbstract = (NettyRemotingAbstract) remotingServer; + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(null); + remotingCommand.setOpaque(command.getOpaque()); + remotingCommand.setCode(proxyOutResult.getCode()); + remotingCommand.setRemark(proxyOutResult.getRemark()); + if (proxyOutResult.getCode() == ResponseCode.SUCCESS && proxyOutResult.getResult() != null) { + ConsumeMessageDirectlyResult consumeMessageDirectlyResult = proxyOutResult.getResult(); + remotingCommand.setBody(consumeMessageDirectlyResult.encode()); + } + SimpleChannel simpleChannel = new SimpleChannel(context.getRemoteAddress(), context.getLocalAddress()); + nettyRemotingAbstract.processResponseCommand(simpleChannel.getChannelHandlerContext(), remotingCommand); + } + }); + return future; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyChannel.java new file mode 100644 index 0000000000..315f69f5ec --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyChannel.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelConfig; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelId; +import io.netty.channel.ChannelMetadata; +import io.netty.channel.ChannelOutboundBuffer; +import io.netty.channel.DefaultChannelPromise; +import io.netty.channel.EventLoop; +import io.netty.util.concurrent.GlobalEventExecutor; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.channel.SimpleChannel; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public abstract class ProxyChannel extends SimpleChannel { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + protected final SocketAddress remoteSocketAddress; + protected final SocketAddress localSocketAddress; + + protected final ProxyRelayService proxyRelayService; + + protected ProxyChannel(ProxyRelayService proxyRelayService, Channel parent, String remoteAddress, + String localAddress) { + super(parent, remoteAddress, localAddress); + this.proxyRelayService = proxyRelayService; + this.remoteSocketAddress = RemotingUtil.string2SocketAddress(remoteAddress); + this.localSocketAddress = RemotingUtil.string2SocketAddress(localAddress); + } + + protected ProxyChannel(ProxyRelayService proxyRelayService, Channel parent, ChannelId id, String remoteAddress, + String localAddress) { + super(parent, id, remoteAddress, localAddress); + this.proxyRelayService = proxyRelayService; + this.remoteSocketAddress = RemotingUtil.string2SocketAddress(remoteAddress); + this.localSocketAddress = RemotingUtil.string2SocketAddress(localAddress); + } + + @Override + public ChannelFuture writeAndFlush(Object msg) { + CompletableFuture processFuture = new CompletableFuture<>(); + + try { + if (msg instanceof RemotingCommand) { + ProxyContext context = ProxyContext.createForInner(this.getClass()) + .setRemoteAddress(remoteAddress) + .setLocalAddress(localAddress); + RemotingCommand command = (RemotingCommand) msg; + if (command.getExtFields() == null) { + command.setExtFields(new HashMap<>()); + } + switch (command.getCode()) { + case RequestCode.CHECK_TRANSACTION_STATE: { + CheckTransactionStateRequestHeader header = (CheckTransactionStateRequestHeader) command.readCustomHeader(); + MessageExt messageExt = MessageDecoder.decode(ByteBuffer.wrap(command.getBody()), true, false, false); + RelayData relayData = this.proxyRelayService.processCheckTransactionState(context, command, header, messageExt); + processFuture = this.processCheckTransaction(header, messageExt, relayData.getProcessResult(), relayData.getRelayFuture()); + break; + } + case RequestCode.GET_CONSUMER_RUNNING_INFO: { + GetConsumerRunningInfoRequestHeader header = (GetConsumerRunningInfoRequestHeader) command.readCustomHeader(); + CompletableFuture> relayFuture = this.proxyRelayService.processGetConsumerRunningInfo(context, command, header); + processFuture = this.processGetConsumerRunningInfo(command, header, relayFuture); + break; + } + case RequestCode.CONSUME_MESSAGE_DIRECTLY: { + ConsumeMessageDirectlyResultRequestHeader header = (ConsumeMessageDirectlyResultRequestHeader) command.readCustomHeader(); + MessageExt messageExt = MessageDecoder.decode(ByteBuffer.wrap(command.getBody()), true, false, false); + processFuture = this.processConsumeMessageDirectly(command, header, messageExt, + this.proxyRelayService.processConsumeMessageDirectly(context, command, header)); + break; + } + default: + break; + } + } else { + processFuture = processOtherMessage(msg); + } + } catch (Throwable t) { + log.error("process failed. msg:{}", msg, t); + processFuture.completeExceptionally(t); + } + + DefaultChannelPromise promise = new DefaultChannelPromise(this, GlobalEventExecutor.INSTANCE); + processFuture.thenAccept(ignore -> promise.setSuccess()) + .exceptionally(t -> { + promise.setFailure(t); + return null; + }); + return promise; + } + + protected abstract CompletableFuture processOtherMessage(Object msg); + + protected abstract CompletableFuture processCheckTransaction( + CheckTransactionStateRequestHeader header, + MessageExt messageExt, + TransactionData transactionData, + CompletableFuture> responseFuture); + + protected abstract CompletableFuture processGetConsumerRunningInfo( + RemotingCommand command, + GetConsumerRunningInfoRequestHeader header, + CompletableFuture> responseFuture); + + protected abstract CompletableFuture processConsumeMessageDirectly( + RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header, + MessageExt messageExt, + CompletableFuture> responseFuture); + + @Override + public ChannelConfig config() { + return null; + } + + @Override + public ChannelMetadata metadata() { + return null; + } + + @Override + protected AbstractUnsafe newUnsafe() { + return null; + } + + @Override + protected boolean isCompatible(EventLoop loop) { + return false; + } + + @Override + protected void doBind(SocketAddress localAddress) throws Exception { + + } + + @Override + protected void doDisconnect() throws Exception { + + } + + @Override + protected void doClose() throws Exception { + + } + + @Override + protected void doBeginRead() throws Exception { + + } + + @Override + protected void doWrite(ChannelOutboundBuffer in) throws Exception { + + } + + @Override + protected SocketAddress localAddress0() { + return this.localSocketAddress; + } + + @Override + protected SocketAddress remoteAddress0() { + return this.remoteSocketAddress; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayResult.java new file mode 100644 index 0000000000..95b98d4d6b --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayResult.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +public class ProxyRelayResult { + private int code; + private String remark; + private T result; + + public ProxyRelayResult(int code, String remark, T result) { + this.code = code; + this.remark = remark; + this.result = result; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getRemark() { + return remark; + } + + public void setRemark(String remark) { + this.remark = remark; + } + + public T getResult() { + return result; + } + + public void setResult(T result) { + this.result = result; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayService.java new file mode 100644 index 0000000000..9785f14ddd --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/ProxyRelayService.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; + +public interface ProxyRelayService { + + CompletableFuture> processGetConsumerRunningInfo( + ProxyContext context, + RemotingCommand command, + GetConsumerRunningInfoRequestHeader header + ); + + CompletableFuture> processConsumeMessageDirectly( + ProxyContext context, + RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header + ); + + RelayData processCheckTransactionState( + ProxyContext context, + RemotingCommand command, + CheckTransactionStateRequestHeader header, + MessageExt messageExt + ); +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/RelayData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/RelayData.java new file mode 100644 index 0000000000..20ee0f5fdf --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/relay/RelayData.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; + +public class RelayData { + private T processResult; + private CompletableFuture> relayFuture; + + public RelayData(T processResult, CompletableFuture> relayFuture) { + this.processResult = processResult; + this.relayFuture = relayFuture; + } + + public CompletableFuture> getRelayFuture() { + return relayFuture; + } + + public void setRelayFuture( + CompletableFuture> relayFuture) { + this.relayFuture = relayFuture; + } + + public T getProcessResult() { + return processResult; + } + + public void setProcessResult(T processResult) { + this.processResult = processResult; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/AddressableMessageQueue.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/AddressableMessageQueue.java new file mode 100644 index 0000000000..ca877f3278 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/AddressableMessageQueue.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.base.MoreObjects; +import java.util.Objects; +import org.apache.rocketmq.common.message.MessageQueue; + +public class AddressableMessageQueue implements Comparable { + + private final MessageQueue messageQueue; + private final String brokerAddr; + + public AddressableMessageQueue(MessageQueue messageQueue, String brokerAddr) { + this.messageQueue = messageQueue; + this.brokerAddr = brokerAddr; + } + + @Override + public int compareTo(AddressableMessageQueue o) { + return messageQueue.compareTo(o.messageQueue); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AddressableMessageQueue)) { + return false; + } + AddressableMessageQueue queue = (AddressableMessageQueue) o; + return Objects.equals(messageQueue, queue.messageQueue); + } + + @Override + public int hashCode() { + return messageQueue == null ? 1 : messageQueue.hashCode(); + } + + public int getQueueId() { + return this.messageQueue.getQueueId(); + } + + public String getBrokerName() { + return this.messageQueue.getBrokerName(); + } + + public String getTopic() { + return messageQueue.getTopic(); + } + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + public String getBrokerAddr() { + return brokerAddr; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("messageQueue", messageQueue) + .add("brokerAddr", brokerAddr) + .toString(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java new file mode 100644 index 0000000000..e7fcd1f9b5 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteService.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import java.util.List; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; + +public class ClusterTopicRouteService extends TopicRouteService { + + public ClusterTopicRouteService(MQClientAPIFactory mqClientAPIFactory) { + super(mqClientAPIFactory); + } + + @Override + public MessageQueueView getCurrentMessageQueueView(String topicName) throws Exception { + return getAllMessageQueueView(topicName); + } + + @Override + public ProxyTopicRouteData getTopicRouteForProxy(List
requestHostAndPortList, + String topicName) throws Exception { + TopicRouteData topicRouteData = getAllMessageQueueView(topicName).getTopicRouteData(); + + ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData(); + proxyTopicRouteData.setQueueDatas(topicRouteData.getQueueDatas()); + + for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { + ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData(); + proxyBrokerData.setCluster(brokerData.getCluster()); + proxyBrokerData.setBrokerName(brokerData.getBrokerName()); + for (Long brokerId : brokerData.getBrokerAddrs().keySet()) { + proxyBrokerData.getBrokerAddrs().put(brokerId, requestHostAndPortList); + } + proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData); + } + + return proxyTopicRouteData; + } + + @Override + public String getBrokerAddr(String brokerName) throws Exception { + List brokerDataList = getAllMessageQueueView(brokerName).getTopicRouteData().getBrokerDatas(); + if (brokerDataList.isEmpty()) { + return null; + } + return brokerDataList.get(0).getBrokerAddrs().get(MixAll.MASTER_ID); + } + + @Override + public AddressableMessageQueue buildAddressableMessageQueue(MessageQueue messageQueue) throws Exception { + String brokerAddress = getBrokerAddr(messageQueue.getBrokerName()); + return new AddressableMessageQueue(messageQueue, brokerAddress); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java new file mode 100644 index 0000000000..2ff1f3cd0a --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.collect.Lists; +import com.google.common.net.HostAndPort; +import java.util.HashMap; +import java.util.List; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; + +public class LocalTopicRouteService extends TopicRouteService { + + private final BrokerController brokerController; + private final List brokerDataList; + private final int grpcPort; + + public LocalTopicRouteService(BrokerController brokerController, MQClientAPIFactory mqClientAPIFactory) { + super(mqClientAPIFactory); + this.brokerController = brokerController; + BrokerConfig brokerConfig = this.brokerController.getBrokerConfig(); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, this.brokerController.getBrokerAddr()); + this.brokerDataList = Lists.newArrayList( + new BrokerData(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), brokerAddrs) + ); + this.grpcPort = ConfigurationManager.getProxyConfig().getGrpcServerPort(); + } + + @Override + public MessageQueueView getCurrentMessageQueueView(String topic) throws Exception { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); + return new MessageQueueView(topic, toTopicRouteData(topicConfig)); + } + + @Override + public ProxyTopicRouteData getTopicRouteForProxy(List
requestHostAndPortList, + String topicName) throws Exception { + MessageQueueView messageQueueView = getAllMessageQueueView(topicName); + TopicRouteData topicRouteData = messageQueueView.getTopicRouteData(); + + ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData(); + proxyTopicRouteData.setQueueDatas(topicRouteData.getQueueDatas()); + + for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { + ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData(); + proxyBrokerData.setCluster(brokerData.getCluster()); + proxyBrokerData.setBrokerName(brokerData.getBrokerName()); + for (Long brokerId : brokerData.getBrokerAddrs().keySet()) { + String brokerAddr = brokerData.getBrokerAddrs().get(brokerId); + HostAndPort brokerHostAndPort = HostAndPort.fromString(brokerAddr); + HostAndPort grpcHostAndPort = HostAndPort.fromParts(brokerHostAndPort.getHost(), grpcPort); + + proxyBrokerData.getBrokerAddrs().put(brokerId, Lists.newArrayList(new Address(Address.AddressScheme.IPv4, grpcHostAndPort))); + } + proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData); + } + + return proxyTopicRouteData; + } + + @Override + public String getBrokerAddr(String brokerName) throws Exception { + return this.brokerController.getBrokerAddr(); + } + + @Override + public AddressableMessageQueue buildAddressableMessageQueue(MessageQueue messageQueue) throws Exception { + String brokerAddress = getBrokerAddr(messageQueue.getBrokerName()); + return new AddressableMessageQueue(messageQueue, brokerAddress); + } + + protected TopicRouteData toTopicRouteData(TopicConfig topicConfig) { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setBrokerDatas(brokerDataList); + + QueueData queueData = new QueueData(); + queueData.setPerm(topicConfig.getPerm()); + queueData.setReadQueueNums(topicConfig.getReadQueueNums()); + queueData.setWriteQueueNums(topicConfig.getWriteQueueNums()); + queueData.setTopicSysFlag(topicConfig.getTopicSysFlag()); + queueData.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + topicRouteData.setQueueDatas(Lists.newArrayList(queueData)); + + return topicRouteData; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java new file mode 100644 index 0000000000..cf5314c79f --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.base.MoreObjects; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.route.QueueData; + +public class MessageQueueSelector { + private static final int BROKER_ACTING_QUEUE_ID = -1; + + // multiple queues for brokers with queueId : normal + private final List queues = new ArrayList<>(); + // one queue for brokers with queueId : -1 + private final List brokerActingQueues = new ArrayList<>(); + private final Map brokerNameQueueMap = new ConcurrentHashMap<>(); + private final AtomicInteger queueIndex; + private final AtomicInteger brokerIndex; + + public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { + if (read) { + this.queues.addAll(buildRead(topicRouteWrapper)); + } else { + this.queues.addAll(buildWrite(topicRouteWrapper)); + } + buildBrokerActingQueues(topicRouteWrapper.getTopicName(), this.queues); + + this.queueIndex = new AtomicInteger(Math.abs(new Random().nextInt())); + this.brokerIndex = new AtomicInteger(Math.abs(new Random().nextInt())); + } + + private static List buildRead(TopicRouteWrapper topicRoute) { + Set queueSet = new HashSet<>(); + List qds = topicRoute.getQueueDatas(); + if (qds == null) { + return new ArrayList<>(); + } + + for (QueueData qd : qds) { + if (PermName.isReadable(qd.getPerm())) { + String brokerAddr = topicRoute.getMasterAddrPrefer(qd.getBrokerName()); + if (brokerAddr == null) { + continue; + } + + for (int i = 0; i < qd.getReadQueueNums(); i++) { + AddressableMessageQueue mq = new AddressableMessageQueue( + new MessageQueue(topicRoute.getTopicName(), qd.getBrokerName(), i), + brokerAddr); + queueSet.add(mq); + } + } + } + + return queueSet.stream().sorted().collect(Collectors.toList()); + } + + private static List buildWrite(TopicRouteWrapper topicRoute) { + Set queueSet = new HashSet<>(); + // order topic route. + if (StringUtils.isNotBlank(topicRoute.getOrderTopicConf())) { + String[] brokers = topicRoute.getOrderTopicConf().split(";"); + for (String broker : brokers) { + String[] item = broker.split(":"); + String brokerName = item[0]; + String brokerAddr = topicRoute.getMasterAddr(brokerName); + if (brokerAddr == null) { + continue; + } + + int nums = Integer.parseInt(item[1]); + for (int i = 0; i < nums; i++) { + AddressableMessageQueue mq = new AddressableMessageQueue( + new MessageQueue(topicRoute.getTopicName(), brokerName, i), + brokerAddr); + queueSet.add(mq); + } + } + } else { + List qds = topicRoute.getQueueDatas(); + if (qds == null) { + return new ArrayList<>(); + } + + for (QueueData qd : qds) { + if (PermName.isWriteable(qd.getPerm())) { + String brokerAddr = topicRoute.getMasterAddr(qd.getBrokerName()); + if (brokerAddr == null) { + continue; + } + + for (int i = 0; i < qd.getWriteQueueNums(); i++) { + AddressableMessageQueue mq = new AddressableMessageQueue( + new MessageQueue(topicRoute.getTopicName(), qd.getBrokerName(), i), + brokerAddr); + queueSet.add(mq); + } + } + } + } + + return queueSet.stream().sorted().collect(Collectors.toList()); + } + + private void buildBrokerActingQueues(String topic, List normalQueues) { + for (AddressableMessageQueue mq : normalQueues) { + AddressableMessageQueue brokerActingQueue = new AddressableMessageQueue( + new MessageQueue(topic, mq.getMessageQueue().getBrokerName(), BROKER_ACTING_QUEUE_ID), + mq.getBrokerAddr()); + + if (!brokerActingQueues.contains(brokerActingQueue)) { + brokerActingQueues.add(brokerActingQueue); + brokerNameQueueMap.put(brokerActingQueue.getBrokerName(), brokerActingQueue); + } + } + + Collections.sort(brokerActingQueues); + } + + public AddressableMessageQueue getQueueByBrokerName(String brokerName) { + return this.brokerNameQueueMap.get(brokerName); + } + + public AddressableMessageQueue selectOne(boolean onlyBroker) { + int nextIndex = onlyBroker ? brokerIndex.getAndIncrement() : queueIndex.getAndIncrement(); + return selectOneByIndex(nextIndex, onlyBroker); + } + + public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) { + boolean onlyBroker = last.getQueueId() < 0; + AddressableMessageQueue newOne = last; + int count = onlyBroker ? brokerActingQueues.size() : queues.size(); + + for (int i = 0; i < count; i++) { + newOne = selectOne(onlyBroker); + if (!newOne.getBrokerName().equals(last.getBrokerName()) || newOne.getQueueId() != last.getQueueId()) { + break; + } + } + return newOne; + } + + public AddressableMessageQueue selectOneByIndex(int index, boolean onlyBroker) { + if (onlyBroker) { + if (brokerActingQueues.isEmpty()) { + return null; + } + return brokerActingQueues.get(Math.abs(index) % brokerActingQueues.size()); + } + + if (queues.isEmpty()) { + return null; + } + return queues.get(Math.abs(index) % queues.size()); + } + + public List getQueues() { + return queues; + } + + public List getBrokerActingQueues() { + return brokerActingQueues; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MessageQueueSelector)) { + return false; + } + MessageQueueSelector queue = (MessageQueueSelector) o; + return Objects.equals(queues, queue.queues) && + Objects.equals(brokerActingQueues, queue.brokerActingQueues); + } + + @Override + public int hashCode() { + return Objects.hash(queues, brokerActingQueues); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("queues", queues) + .add("brokerActingQueues", brokerActingQueues) + .add("brokerNameQueueMap", brokerNameQueueMap) + .add("queueIndex", queueIndex) + .add("brokerIndex", brokerIndex) + .toString(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java new file mode 100644 index 0000000000..cdef39cc2d --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.base.MoreObjects; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; + +public class MessageQueueView { + public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData()); + + private final MessageQueueSelector readSelector; + private final MessageQueueSelector writeSelector; + private final TopicRouteWrapper topicRouteWrapper; + + public MessageQueueView(String topic, TopicRouteData topicRouteData) { + this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); + + this.readSelector = new MessageQueueSelector(topicRouteWrapper, true); + this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false); + } + + public TopicRouteData getTopicRouteData() { + return topicRouteWrapper.getTopicRouteData(); + } + + public String getTopicName() { + return topicRouteWrapper.getTopicName(); + } + + public boolean isEmptyCachedQueue() { + return this == WRAPPED_EMPTY_QUEUE; + } + + public MessageQueueSelector getReadSelector() { + return readSelector; + } + + public MessageQueueSelector getWriteSelector() { + return writeSelector; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("readSelector", readSelector) + .add("writeSelector", writeSelector) + .add("topicRouteWrapper", topicRouteWrapper) + .toString(); + } +} \ No newline at end of file diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java new file mode 100644 index 0000000000..92931589d8 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/ProxyTopicRouteData.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.proxy.common.Address; + +public class ProxyTopicRouteData { + + public static class ProxyBrokerData { + private String cluster; + private String brokerName; + private Map/* broker address */> brokerAddrs = new HashMap<>(); + + public String getCluster() { + return cluster; + } + + public void setCluster(String cluster) { + this.cluster = cluster; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + public Map> getBrokerAddrs() { + return brokerAddrs; + } + + public void setBrokerAddrs(Map> brokerAddrs) { + this.brokerAddrs = brokerAddrs; + } + + public BrokerData buildBrokerData() { + BrokerData brokerData = new BrokerData(); + brokerData.setCluster(cluster); + brokerData.setBrokerName(brokerName); + HashMap buildBrokerAddress = new HashMap<>(); + brokerAddrs.forEach((k, v) -> { + if (!v.isEmpty()) { + buildBrokerAddress.put(k, v.get(0).getHostAndPort().toString()); + } + }); + brokerData.setBrokerAddrs(buildBrokerAddress); + return brokerData; + } + } + + private List queueDatas = new ArrayList<>(); + private List brokerDatas = new ArrayList<>(); + + public List getQueueDatas() { + return queueDatas; + } + + public void setQueueDatas(List queueDatas) { + this.queueDatas = queueDatas; + } + + public List getBrokerDatas() { + return brokerDatas; + } + + public void setBrokerDatas(List brokerDatas) { + this.brokerDatas = brokerDatas; + } + + public TopicRouteData buildTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setQueueDatas(queueDatas); + topicRouteData.setBrokerDatas(brokerDatas.stream() + .map(ProxyBrokerData::buildBrokerData) + .collect(Collectors.toList())); + return topicRouteData; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteHelper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteHelper.java new file mode 100644 index 0000000000..2df41255df --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteHelper.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import org.apache.rocketmq.client.common.ClientErrorCode; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.protocol.ResponseCode; + +public class TopicRouteHelper { + + public static boolean isTopicNotExistError(Throwable e) { + if (e instanceof MQBrokerException) { + if (((MQBrokerException) e).getResponseCode() == ResponseCode.TOPIC_NOT_EXIST) { + return true; + } + } + + if (e instanceof MQClientException) { + int code = ((MQClientException) e).getResponseCode(); + if (code == ResponseCode.TOPIC_NOT_EXIST || code == ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION) { + return true; + } + + Throwable cause = e.getCause(); + if (cause instanceof MQClientException) { + int causeCode = ((MQClientException) cause).getResponseCode(); + return causeCode == ResponseCode.TOPIC_NOT_EXIST || causeCode == ClientErrorCode.NOT_FOUND_TOPIC_EXCEPTION; + } + } + + return false; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java new file mode 100644 index 0000000000..d8909a3f10 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.LoadingCache; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.ThreadFactoryImpl; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.AbstractCacheLoader; +import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; + +public abstract class TopicRouteService extends AbstractStartAndShutdown { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private final MQClientAPIFactory mqClientAPIFactory; + + protected final LoadingCache topicCache; + protected final ScheduledExecutorService scheduledExecutorService; + protected final ThreadPoolExecutor cacheRefreshExecutor; + + public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { + ProxyConfig config = ConfigurationManager.getProxyConfig(); + + this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryImpl("TopicRouteService_") + ); + this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( + config.getTopicRouteServiceThreadPoolNums(), + config.getTopicRouteServiceThreadPoolNums(), + 1000 * 60, + TimeUnit.MILLISECONDS, + "TopicRouteCacheRefresh", + config.getTopicRouteServiceThreadPoolQueueCapacity() + ); + this.mqClientAPIFactory = mqClientAPIFactory; + this.topicCache = CacheBuilder.newBuilder() + .maximumSize(config.getTopicRouteServiceCacheMaxNum()) + .refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS) + .build(new TopicRouteCacheLoader()); + + this.init(); + } + + protected void init() { + this.appendShutdown(this.scheduledExecutorService::shutdown); + this.appendStartAndShutdown(this.mqClientAPIFactory); + } + + public MessageQueueView getAllMessageQueueView(String topicName) throws Exception { + return getCacheMessageQueueWrapper(this.topicCache, topicName); + } + + public abstract MessageQueueView getCurrentMessageQueueView(String topicName) throws Exception; + + public abstract ProxyTopicRouteData getTopicRouteForProxy(List
requestHostAndPortList, + String topicName) throws Exception; + + public abstract String getBrokerAddr(String brokerName) throws Exception; + + public abstract AddressableMessageQueue buildAddressableMessageQueue(MessageQueue messageQueue) throws Exception; + + protected static MessageQueueView getCacheMessageQueueWrapper(LoadingCache topicCache, + String key) throws Exception { + MessageQueueView res = topicCache.get(key); + if (res.isEmptyCachedQueue()) { + throw new MQClientException(ResponseCode.TOPIC_NOT_EXIST, + "No topic route info in name server for the topic: " + key); + } + return res; + } + + protected static boolean isTopicRouteValid(TopicRouteData routeData) { + return routeData != null && routeData.getQueueDatas() != null && !routeData.getQueueDatas().isEmpty() + && routeData.getBrokerDatas() != null && !routeData.getBrokerDatas().isEmpty(); + } + + protected abstract class AbstractTopicRouteCacheLoader extends AbstractCacheLoader { + + public AbstractTopicRouteCacheLoader() { + super(cacheRefreshExecutor); + } + + protected abstract TopicRouteData loadTopicRouteData(String topic) throws Exception; + + @Override + public MessageQueueView getDirectly(String topic) throws Exception { + try { + TopicRouteData topicRouteData = loadTopicRouteData(topic); + + if (isTopicRouteValid(topicRouteData)) { + MessageQueueView tmp = new MessageQueueView(topic, topicRouteData); + log.info("load topic route from namesrv. topic: {}, queue: {}", topic, tmp); + return tmp; + } + return MessageQueueView.WRAPPED_EMPTY_QUEUE; + } catch (Exception e) { + if (TopicRouteHelper.isTopicNotExistError(e)) { + return MessageQueueView.WRAPPED_EMPTY_QUEUE; + } + throw e; + } + } + + @Override + protected void onErr(String key, Exception e) { + log.error("load topic route from namesrv failed. topic:{}", key, e); + } + } + + protected class TopicRouteCacheLoader extends AbstractTopicRouteCacheLoader { + + @Override + protected TopicRouteData loadTopicRouteData(String topic) throws Exception { + return mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); + } + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteWrapper.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteWrapper.java new file mode 100644 index 0000000000..3950d92a1d --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteWrapper.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.route; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; + +public class TopicRouteWrapper { + + private final TopicRouteData topicRouteData; + private final String topicName; + private final Map brokerNameRouteData = new HashMap<>(); + + public TopicRouteWrapper(TopicRouteData topicRouteData, String topicName) { + this.topicRouteData = topicRouteData; + this.topicName = topicName; + + if (this.topicRouteData.getBrokerDatas() != null) { + for (BrokerData brokerData : this.topicRouteData.getBrokerDatas()) { + this.brokerNameRouteData.put(brokerData.getBrokerName(), brokerData); + } + } + } + + public String getMasterAddr(String brokerName) { + return this.brokerNameRouteData.get(brokerName).getBrokerAddrs().get(MixAll.MASTER_ID); + } + + public String getMasterAddrPrefer(String brokerName) { + HashMap brokerAddr = brokerNameRouteData.get(brokerName).getBrokerAddrs(); + String addr = brokerAddr.get(MixAll.MASTER_ID); + if (addr == null) { + Optional optional = brokerAddr.keySet().stream().findFirst(); + return optional.map(brokerAddr::get).orElse(null); + } + return addr; + } + + public String getTopicName() { + return topicName; + } + + public TopicRouteData getTopicRouteData() { + return topicRouteData; + } + + public List getQueueDatas() { + return this.topicRouteData.getQueueDatas(); + } + + public String getOrderTopicConf() { + return this.topicRouteData.getOrderTopicConf(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java new file mode 100644 index 0000000000..262e430c30 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionService.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; + +public abstract class AbstractTransactionService implements TransactionService, StartAndShutdown { + + protected TransactionDataManager transactionDataManager = new TransactionDataManager(); + + @Override + public TransactionData addTransactionDataByBrokerAddr(String brokerAddr, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId, + Message message) { + return this.addTransactionDataByBrokerName(this.getBrokerNameByAddr(brokerAddr), producerGroup, tranStateTableOffset, commitLogOffset, transactionId, message); + } + + @Override + public TransactionData addTransactionDataByBrokerName(String brokerName, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId, + Message message) { + if (StringUtils.isBlank(brokerName)) { + return null; + } + TransactionData transactionData = new TransactionData( + brokerName, + tranStateTableOffset, commitLogOffset, transactionId, + System.currentTimeMillis(), + ConfigurationManager.getProxyConfig().getTransactionDataExpireMillis()); + + this.transactionDataManager.addTransactionData( + producerGroup, + transactionId, + transactionData + ); + return transactionData; + } + + @Override + public EndTransactionRequestData genEndTransactionRequestHeader(String producerGroup, Integer commitOrRollback, + boolean fromTransactionCheck, String msgId, String transactionId) { + TransactionData transactionData = this.transactionDataManager.pollFirstNoExpireTransactionData(producerGroup, transactionId); + if (transactionData == null) { + return null; + } + EndTransactionRequestHeader header = new EndTransactionRequestHeader(); + header.setProducerGroup(producerGroup); + header.setCommitOrRollback(commitOrRollback); + header.setFromTransactionCheck(fromTransactionCheck); + header.setMsgId(msgId); + header.setTransactionId(transactionId); + header.setTranStateTableOffset(transactionData.getTranStateTableOffset()); + header.setCommitLogOffset(transactionData.getCommitLogOffset()); + return new EndTransactionRequestData(transactionData.getBrokerName(), header); + } + + @Override + public void onSendCheckTransactionStateFailed(ProxyContext context, String producerGroup, TransactionData transactionData) { + this.transactionDataManager.removeTransactionData(producerGroup, transactionData.getTransactionId(), transactionData); + } + + protected abstract String getBrokerNameByAddr(String brokerAddr); + + @Override + public void shutdown() throws Exception { + this.transactionDataManager.shutdown(); + } + + @Override + public void start() throws Exception { + this.transactionDataManager.start(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionService.java new file mode 100644 index 0000000000..48cdab33d1 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionService.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.transaction; + +import com.google.common.collect.Sets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.common.protocol.heartbeat.ProducerData; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.thread.ThreadPoolMonitor; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.ProxyConfig; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.remoting.RPCHook; + +public class ClusterTransactionService extends AbstractTransactionService { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + private static final String TRANS_HEARTBEAT_CLIENT_ID = "rmq-proxy-producer-client"; + + private final MQClientAPIFactory mqClientAPIFactory; + private final TopicRouteService topicRouteService; + + private ThreadPoolExecutor heartbeatExecutors; + private final Map/* cluster list */> groupClusterData = new ConcurrentHashMap<>(); + private final AtomicReference> brokerAddrNameMapRef = new AtomicReference<>(); + private TxHeartbeatServiceThread txHeartbeatServiceThread; + + public ClusterTransactionService(TopicRouteService topicRouteService, ProducerManager producerManager, + RPCHook rpcHook, + MQClientAPIFactory mqClientAPIFactory) { + this.topicRouteService = topicRouteService; + this.mqClientAPIFactory = mqClientAPIFactory; + } + + @Override + public void addTransactionSubscription(String group, List topicList) { + for (String topic : topicList) { + addTransactionSubscription(group, topic); + } + } + + @Override + public void addTransactionSubscription(String group, String topic) { + try { + groupClusterData.compute(group, (groupName, clusterDataSet) -> { + if (clusterDataSet == null) { + clusterDataSet = Sets.newHashSet(); + } + clusterDataSet.addAll(getClusterDataFromTopic(topic)); + return clusterDataSet; + }); + } catch (Exception e) { + log.error("add producer group err in txHeartBeat. groupId: {}, err: {}", group, e); + } + } + + @Override + public void replaceTransactionSubscription(String group, List topicList) { + Set clusterDataSet = new HashSet<>(); + for (String topic : topicList) { + clusterDataSet.addAll(getClusterDataFromTopic(topic)); + } + groupClusterData.put(group, clusterDataSet); + } + + private Set getClusterDataFromTopic(String topic) { + try { + MessageQueueView messageQueue = this.topicRouteService.getAllMessageQueueView(topic); + List brokerDataList = messageQueue.getTopicRouteData().getBrokerDatas(); + + if (brokerDataList == null) { + return Collections.emptySet(); + } + Set res = Sets.newHashSet(); + for (BrokerData brokerData : brokerDataList) { + res.add(new ClusterData(brokerData.getCluster())); + } + return res; + } catch (Throwable t) { + log.error("get cluster data failed in txHeartBeat. topic: {}, err: {}", topic, t); + } + return Collections.emptySet(); + } + + @Override + public void unSubscribeAllTransactionTopic(String group) { + groupClusterData.remove(group); + } + + public void scanProducerHeartBeat() { + Set groupSet = groupClusterData.keySet(); + + Map> clusterHeartbeatData = new HashMap<>(); + for (String group : groupSet) { + groupClusterData.computeIfPresent(group, (groupName, clusterDataSet) -> { + if (clusterDataSet.isEmpty()) { + return null; + } + + ProducerData producerData = new ProducerData(); + producerData.setGroupName(groupName); + + for (ClusterData clusterData : clusterDataSet) { + List heartbeatDataList = clusterHeartbeatData.get(clusterData.cluster); + if (heartbeatDataList == null) { + heartbeatDataList = new ArrayList<>(); + } + + HeartbeatData heartbeatData; + if (heartbeatDataList.isEmpty()) { + heartbeatData = new HeartbeatData(); + heartbeatData.setClientID(TRANS_HEARTBEAT_CLIENT_ID); + heartbeatDataList.add(heartbeatData); + } else { + heartbeatData = heartbeatDataList.get(heartbeatDataList.size() - 1); + if (heartbeatData.getProducerDataSet().size() >= ConfigurationManager.getProxyConfig().getTransactionHeartbeatBatchNum()) { + heartbeatData = new HeartbeatData(); + heartbeatData.setClientID(TRANS_HEARTBEAT_CLIENT_ID); + heartbeatDataList.add(heartbeatData); + } + } + + heartbeatData.getProducerDataSet().add(producerData); + clusterHeartbeatData.put(clusterData.cluster, heartbeatDataList); + } + + if (clusterDataSet.isEmpty()) { + return null; + } + return clusterDataSet; + }); + } + + if (clusterHeartbeatData.isEmpty()) { + return; + } + Set>> clusterEntry = clusterHeartbeatData.entrySet(); + for (Map.Entry> entry : clusterEntry) { + sendHeartBeatToCluster(entry.getKey(), entry.getValue()); + } + } + + public Map> getGroupClusterData() { + return groupClusterData; + } + + protected void sendHeartBeatToCluster(String clusterName, List heartbeatDataList) { + if (heartbeatDataList == null) { + return; + } + Map brokerAddrNameMap = new ConcurrentHashMap<>(); + for (HeartbeatData heartbeatData : heartbeatDataList) { + sendHeartBeatToCluster(clusterName, heartbeatData, brokerAddrNameMap); + } + this.brokerAddrNameMapRef.set(brokerAddrNameMap); + } + + protected void sendHeartBeatToCluster(String clusterName, HeartbeatData heartbeatData, Map brokerAddrNameMap) { + try { + MessageQueueView messageQueue = this.topicRouteService.getAllMessageQueueView(clusterName); + List brokerDataList = messageQueue.getTopicRouteData().getBrokerDatas(); + if (brokerDataList == null) { + return; + } + for (BrokerData brokerData : brokerDataList) { + brokerAddrNameMap.put(brokerData.selectBrokerAddr(), brokerData.getBrokerName()); + heartbeatExecutors.submit(() -> { + String brokerAddr = brokerData.selectBrokerAddr(); + this.mqClientAPIFactory.getClient() + .sendHeartbeatOneway(brokerAddr, heartbeatData, Duration.ofSeconds(3).toMillis()) + .exceptionally(t -> { + log.error("Send transactionHeartbeat to broker err. brokerAddr: {}", brokerAddr, t); + return null; + }); + }); + } + } catch (Exception e) { + log.error("get broker add in cluster failed in tx. clusterName: {}", clusterName, e); + } + } + + @Override + protected String getBrokerNameByAddr(String brokerAddr) { + if (StringUtils.isBlank(brokerAddr)) { + return null; + } + return brokerAddrNameMapRef.get().get(brokerAddr); + } + + static class ClusterData { + private final String cluster; + + public ClusterData(String cluster) { + this.cluster = cluster; + } + + public String getCluster() { + return cluster; + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof ClusterData)) { + return super.equals(obj); + } + + ClusterData other = (ClusterData) obj; + return cluster.equals(other.cluster); + } + + @Override + public int hashCode() { + return cluster.hashCode(); + } + } + + class TxHeartbeatServiceThread extends ServiceThread { + + @Override + public String getServiceName() { + return TxHeartbeatServiceThread.class.getName(); + } + + @Override + public void run() { + while (!this.isStopped()) { + this.waitForRunning(TimeUnit.SECONDS.toMillis(ConfigurationManager.getProxyConfig().getTransactionHeartbeatPeriodSecond())); + } + } + + @Override + protected void onWaitEnd() { + scanProducerHeartBeat(); + } + } + + @Override + public void start() throws Exception { + ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); + txHeartbeatServiceThread = new TxHeartbeatServiceThread(); + + super.start(); + txHeartbeatServiceThread.start(); + heartbeatExecutors = ThreadPoolMonitor.createAndMonitor( + proxyConfig.getTransactionHeartbeatThreadPoolNums(), + proxyConfig.getTransactionHeartbeatThreadPoolNums(), + 0L, TimeUnit.MILLISECONDS, + "TransactionHeartbeatRegisterThread", + proxyConfig.getTransactionHeartbeatThreadPoolQueueCapacity() + ); + } + + @Override + public void shutdown() throws Exception { + txHeartbeatServiceThread.shutdown(); + heartbeatExecutors.shutdown(); + super.shutdown(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/EndTransactionRequestData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/EndTransactionRequestData.java new file mode 100644 index 0000000000..b38b4335f4 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/EndTransactionRequestData.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; + +public class EndTransactionRequestData { + private String brokerName; + private EndTransactionRequestHeader requestHeader; + + public EndTransactionRequestData(String brokerName, EndTransactionRequestHeader requestHeader) { + this.brokerName = brokerName; + this.requestHeader = requestHeader; + } + + public String getBrokerName() { + return brokerName; + } + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + public EndTransactionRequestHeader getRequestHeader() { + return requestHeader; + } + + public void setRequestHeader(EndTransactionRequestHeader requestHeader) { + this.requestHeader = requestHeader; + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/LocalTransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/LocalTransactionService.java new file mode 100644 index 0000000000..2371b25a24 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/LocalTransactionService.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.transaction; + +import java.util.List; +import org.apache.rocketmq.common.BrokerConfig; + +/** + * no need to implements, because the channel of producer will put into the broker's producerManager + */ +public class LocalTransactionService extends AbstractTransactionService { + + protected final BrokerConfig brokerConfig; + + public LocalTransactionService(BrokerConfig brokerConfig) { + this.brokerConfig = brokerConfig; + } + + @Override + public void addTransactionSubscription(String group, List topicList) { + + } + + @Override + public void addTransactionSubscription(String group, String topic) { + + } + + @Override + public void replaceTransactionSubscription(String group, List topicList) { + + } + + @Override + public void unSubscribeAllTransactionTopic(String group) { + + } + + @Override + protected String getBrokerNameByAddr(String brokerAddr) { + return this.brokerConfig.getBrokerName(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java new file mode 100644 index 0000000000..88fbf44396 --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionData.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; +import com.google.common.collect.ComparisonChain; + +public class TransactionData implements Comparable { + private final String brokerName; + private final long tranStateTableOffset; + private final long commitLogOffset; + private final String transactionId; + private final long checkTimestamp; + private final long expireMs; + + public TransactionData(String brokerName, long tranStateTableOffset, long commitLogOffset, String transactionId, + long checkTimestamp, long expireMs) { + this.brokerName = brokerName; + this.tranStateTableOffset = tranStateTableOffset; + this.commitLogOffset = commitLogOffset; + this.transactionId = transactionId; + this.checkTimestamp = checkTimestamp; + this.expireMs = expireMs; + } + + public String getBrokerName() { + return brokerName; + } + + public long getTranStateTableOffset() { + return tranStateTableOffset; + } + + public long getCommitLogOffset() { + return commitLogOffset; + } + + public String getTransactionId() { + return transactionId; + } + + public long getCheckTimestamp() { + return checkTimestamp; + } + + public long getExpireMs() { + return expireMs; + } + + public long getExpireTime() { + return checkTimestamp + expireMs; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TransactionData data = (TransactionData) o; + return tranStateTableOffset == data.tranStateTableOffset && commitLogOffset == data.commitLogOffset && + getExpireTime() == data.getExpireTime() && Objects.equal(brokerName, data.brokerName) && + Objects.equal(transactionId, data.transactionId); + } + + @Override + public int hashCode() { + return Objects.hashCode(brokerName, transactionId, tranStateTableOffset, commitLogOffset, getExpireTime()); + } + + @Override + public int compareTo(TransactionData o) { + return ComparisonChain.start() + .compare(getExpireTime(), o.getExpireTime()) + .compare(brokerName, o.brokerName) + .compare(commitLogOffset, o.commitLogOffset) + .compare(tranStateTableOffset, o.tranStateTableOffset) + .compare(transactionId, o.transactionId) + .result(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("brokerName", brokerName) + .add("tranStateTableOffset", tranStateTableOffset) + .add("commitLogOffset", commitLogOffset) + .add("transactionId", transactionId) + .add("checkTimestamp", checkTimestamp) + .add("expireMs", expireMs) + .toString(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManager.java new file mode 100644 index 0000000000..740afab3ae --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManager.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import java.util.Iterator; +import java.util.Map; +import java.util.NavigableSet; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.rocketmq.common.ServiceThread; +import org.apache.rocketmq.common.constant.LoggerName; +import org.apache.rocketmq.logging.InternalLogger; +import org.apache.rocketmq.logging.InternalLoggerFactory; +import org.apache.rocketmq.proxy.common.StartAndShutdown; +import org.apache.rocketmq.proxy.config.ConfigurationManager; + +public class TransactionDataManager implements StartAndShutdown { + private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); + + protected final AtomicLong maxTransactionDataExpireTime = new AtomicLong(System.currentTimeMillis()); + protected final Map> transactionIdDataMap = new ConcurrentHashMap<>(); + protected final TransactionDataCleaner transactionDataCleaner = new TransactionDataCleaner(); + + protected String buildKey(String producerGroup, String transactionId) { + return producerGroup + "@" + transactionId; + } + + public void addTransactionData(String producerGroup, String transactionId, TransactionData transactionData) { + this.transactionIdDataMap.compute(buildKey(producerGroup, transactionId), (key, dataSet) -> { + if (dataSet == null) { + dataSet = new ConcurrentSkipListSet<>(); + } + dataSet.add(transactionData); + if (dataSet.size() > ConfigurationManager.getProxyConfig().getTransactionDataMaxNum()) { + dataSet.pollFirst(); + } + return dataSet; + }); + } + + public TransactionData pollFirstNoExpireTransactionData(String producerGroup, String transactionId) { + AtomicReference res = new AtomicReference<>(); + long currTimestamp = System.currentTimeMillis(); + this.transactionIdDataMap.computeIfPresent(buildKey(producerGroup, transactionId), (key, dataSet) -> { + TransactionData data = dataSet.pollFirst(); + while (data != null && data.getExpireTime() < currTimestamp) { + data = dataSet.pollFirst(); + } + if (data != null) { + res.set(data); + } + if (dataSet.isEmpty()) { + return null; + } + return dataSet; + }); + return res.get(); + } + + public void removeTransactionData(String producerGroup, String transactionId, TransactionData transactionData) { + this.transactionIdDataMap.computeIfPresent(buildKey(producerGroup, transactionId), (key, dataSet) -> { + dataSet.remove(transactionData); + if (dataSet.isEmpty()) { + return null; + } + return dataSet; + }); + } + + protected void cleanExpireTransactionData() { + long currTimestamp = System.currentTimeMillis(); + Set transactionIdSet = this.transactionIdDataMap.keySet(); + for (String transactionId : transactionIdSet) { + this.transactionIdDataMap.computeIfPresent(transactionId, (transactionIdKey, dataSet) -> { + Iterator iterator = dataSet.iterator(); + while (iterator.hasNext()) { + try { + TransactionData data = iterator.next(); + if (data.getExpireTime() < currTimestamp) { + iterator.remove(); + } else { + break; + } + } catch (NoSuchElementException ignore) { + break; + } + } + if (dataSet.isEmpty()) { + return null; + } + try { + TransactionData maxData = dataSet.last(); + maxTransactionDataExpireTime.set(Math.max(maxTransactionDataExpireTime.get(), maxData.getExpireTime())); + } catch (NoSuchElementException ignore) { + } + return dataSet; + }); + } + } + + protected class TransactionDataCleaner extends ServiceThread { + + @Override + public String getServiceName() { + return "TransactionDataCleaner"; + } + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + while (!this.isStopped()) { + this.waitForRunning(ConfigurationManager.getProxyConfig().getTransactionDataExpireScanPeriodMillis()); + } + log.info(this.getServiceName() + " service stopped"); + } + + @Override + protected void onWaitEnd() { + cleanExpireTransactionData(); + } + } + + protected void waitTransactionDataClear() throws InterruptedException { + this.cleanExpireTransactionData(); + long waitMs = Math.max(this.maxTransactionDataExpireTime.get() - System.currentTimeMillis(), 0); + waitMs = Math.min(waitMs, ConfigurationManager.getProxyConfig().getTransactionDataMaxWaitClearMillis()); + + if (waitMs > 0) { + TimeUnit.MILLISECONDS.sleep(waitMs); + } + } + + @Override + public void shutdown() throws Exception { + this.transactionDataCleaner.shutdown(); + this.waitTransactionDataClear(); + } + + @Override + public void start() throws Exception { + this.transactionDataCleaner.start(); + } +} diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java new file mode 100644 index 0000000000..2a851051eb --- /dev/null +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/transaction/TransactionService.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.rocketmq.proxy.service.transaction; + +import java.util.List; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.proxy.common.ProxyContext; + +public interface TransactionService { + + void addTransactionSubscription(String group, List topicList); + + void addTransactionSubscription(String group, String topic); + + void replaceTransactionSubscription(String group, List topicList); + + void unSubscribeAllTransactionTopic(String group); + + TransactionData addTransactionDataByBrokerAddr(String brokerAddr, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId, + Message message); + + TransactionData addTransactionDataByBrokerName(String brokerName, String producerGroup, long tranStateTableOffset, long commitLogOffset, String transactionId, + Message message); + + EndTransactionRequestData genEndTransactionRequestHeader(String producerGroup, Integer commitOrRollback, + boolean fromTransactionCheck, String msgId, String transactionId); + + void onSendCheckTransactionStateFailed(ProxyContext context, String producerGroup, TransactionData transactionData); +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/common/utils/FilterUtilTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/common/utils/FilterUtilTest.java new file mode 100644 index 0000000000..0d36a23c73 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/common/utils/FilterUtilTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.common.utils; + +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class FilterUtilTest { + @Test + public void testTagMatched() throws Exception { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData("", "tagA"); + assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), "tagA")).isTrue(); + } + + @Test + public void testTagNotMatched() throws Exception { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData("", "tagA"); + assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), "tagB")).isFalse(); + } + + @Test + public void testTagMatchedStar() throws Exception { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData("", "*"); + assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), "tagA")).isTrue(); + } + + @Test + public void testTagNotMatchedNull() throws Exception { + SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData("", "tagA"); + assertThat(FilterUtils.isTagMatched(subscriptionData.getTagsSet(), null)).isFalse(); + } + +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/config/ConfigurationManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/config/ConfigurationManagerTest.java new file mode 100644 index 0000000000..ca36f5f20f --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/config/ConfigurationManagerTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import org.apache.rocketmq.proxy.ProxyMode; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ConfigurationManagerTest extends InitConfigAndLoggerTest { + + @Test + public void testInitEnv() { + // configure proxy home by system env. + assertThat(ConfigurationManager.getProxyHome()).isEqualTo(mockProxyHome); + } + + @Test + public void testIntConfig() { + assertThat(ConfigurationManager.getProxyConfig()).isNotNull(); + assertThat(ConfigurationManager.getProxyConfig().getProxyMode()).isEqualToIgnoringCase(ProxyMode.CLUSTER.toString()); + + String brokerConfig = ConfigurationManager.getProxyConfig().getBrokerConfigPath(); + assertThat(brokerConfig).isEqualTo(ConfigurationManager.getProxyHome() + "/conf/broker.conf"); + } + + @Test + public void testGetProxyHome() { + // test configured proxy home + assertThat(ConfigurationManager.getProxyHome()).isEqualTo(mockProxyHome); + } + + @Test + public void testGetProxyConfig() { + assertThat(ConfigurationManager.getProxyConfig()).isNotNull(); + } + +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/config/InitConfigAndLoggerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/config/InitConfigAndLoggerTest.java new file mode 100644 index 0000000000..12e8dd74f6 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/config/InitConfigAndLoggerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; +import java.net.URL; +import org.apache.rocketmq.client.log.ClientLogger; +import org.junit.After; +import org.junit.Before; +import org.slf4j.LoggerFactory; + +import static org.apache.rocketmq.proxy.config.ConfigurationManager.RMQ_PROXY_HOME; + +public class InitConfigAndLoggerTest { + public static String mockProxyHome = "/mock/rmq/proxy/home"; + + @Before + public void before() throws Throwable { + URL mockProxyHomeURL = getClass().getClassLoader().getResource("rmq-proxy-home"); + if (mockProxyHomeURL != null) { + mockProxyHome = mockProxyHomeURL.toURI().getPath(); + } + System.setProperty(RMQ_PROXY_HOME, mockProxyHome); + ConfigurationManager.initEnv(); + ConfigurationManager.intConfig(); + initLogger(); + } + + @After + public void after() { + System.clearProperty(RMQ_PROXY_HOME); + System.clearProperty(ClientLogger.CLIENT_LOG_USESLF4J); + } + + private static void initLogger() throws JoranException { + System.setProperty(ClientLogger.CLIENT_LOG_USESLF4J, "true"); + + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + //https://logback.qos.ch/manual/configuration.html + lc.setPackagingDataEnabled(false); + configurator.doConfigure(mockProxyHome + "/conf/logback_proxy.xml"); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/config/MetricCollectorModeTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/config/MetricCollectorModeTest.java new file mode 100644 index 0000000000..59aa49fe83 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/config/MetricCollectorModeTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.config; + +import org.junit.Assert; +import org.junit.Test; + +public class MetricCollectorModeTest { + + @Test + public void testGetEnumByOrdinal() { + Assert.assertEquals(MetricCollectorMode.OFF, MetricCollectorMode.getEnumByOrdinal(0)); + Assert.assertEquals(MetricCollectorMode.ON, MetricCollectorMode.getEnumByOrdinal(1)); + Assert.assertEquals(MetricCollectorMode.PROXY, MetricCollectorMode.getEnumByOrdinal(2)); + } + +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivityTest.java new file mode 100644 index 0000000000..3dad901daf --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/AbstractMessingActivityTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.Resource; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.topic.TopicValidator; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertThrows; + +public class AbstractMessingActivityTest extends InitConfigAndLoggerTest { + + public static class MockMessingActivity extends AbstractMessingActivity { + + public MockMessingActivity(MessagingProcessor messagingProcessor, + GrpcClientSettingsManager grpcClientSettingsManager, + GrpcChannelManager grpcChannelManager) { + super(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + } + + private AbstractMessingActivity messingActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.messingActivity = new MockMessingActivity(null, null, null); + } + + @Test + public void testValidateTopic() { + assertThrows(GrpcProxyException.class, () -> messingActivity.validateTopic(Resource.newBuilder().build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateTopic(Resource.newBuilder().setName(TopicValidator.RMQ_SYS_TRACE_TOPIC).build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateTopic(Resource.newBuilder().setName("@").build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateTopic(Resource.newBuilder().setName(createString(128)).build())); + messingActivity.validateTopic(Resource.newBuilder().setName(createString(127)).build()); + } + + @Test + public void testValidateConsumer() { + assertThrows(GrpcProxyException.class, () -> messingActivity.validateConsumerGroup(Resource.newBuilder().build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateConsumerGroup(Resource.newBuilder().setName(MixAll.CID_SYS_RMQ_TRANS).build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateConsumerGroup(Resource.newBuilder().setName("@").build())); + assertThrows(GrpcProxyException.class, () -> messingActivity.validateConsumerGroup(Resource.newBuilder().setName(createString(256)).build())); + messingActivity.validateConsumerGroup(Resource.newBuilder().setName(createString(255)).build()); + } + + private static String createString(int len) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < len; i++) { + sb.append('a'); + } + return sb.toString(); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java new file mode 100644 index 0000000000..fde7a8af7c --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/BaseActivityTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2; + +import io.grpc.Metadata; +import java.time.Duration; +import java.util.Random; +import java.util.UUID; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.proxy.common.ContextVariable; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.proxy.processor.ReceiptHandleProcessor; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Ignore +@RunWith(MockitoJUnitRunner.Silent.class) +public class BaseActivityTest extends InitConfigAndLoggerTest { + protected static final Random RANDOM = new Random(); + protected MessagingProcessor messagingProcessor; + protected GrpcClientSettingsManager grpcClientSettingsManager; + protected GrpcChannelManager grpcChannelManager; + protected ProxyRelayService proxyRelayService; + protected ReceiptHandleProcessor receiptHandleProcessor; + + protected static final String REMOTE_ADDR = "192.168.0.1:8080"; + protected static final String LOCAL_ADDR = "127.0.0.1:8080"; + protected Metadata metadata = new Metadata(); + + protected static final String CLIENT_ID = "client-id" + UUID.randomUUID(); + protected static final String JAVA = "JAVA"; + + public void before() throws Throwable { + super.before(); + messagingProcessor = mock(MessagingProcessor.class); + grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); + proxyRelayService = mock(ProxyRelayService.class); + receiptHandleProcessor = mock(ReceiptHandleProcessor.class); + + metadata.put(InterceptorConstants.CLIENT_ID, CLIENT_ID); + metadata.put(InterceptorConstants.LANGUAGE, JAVA); + metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR); + metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR); + when(messagingProcessor.getProxyRelayService()).thenReturn(proxyRelayService); + grpcChannelManager = new GrpcChannelManager(messagingProcessor.getProxyRelayService()); + } + + protected ProxyContext createContext() { + return ProxyContext.create() + .withVal(ContextVariable.CLIENT_ID, CLIENT_ID) + .withVal(ContextVariable.LANGUAGE, JAVA) + .withVal(ContextVariable.REMOTE_ADDRESS, REMOTE_ADDR) + .withVal(ContextVariable.LOCAL_ADDRESS, LOCAL_ADDR) + .withVal(ContextVariable.REMAINING_MS, Duration.ofSeconds(10).toMillis()); + } + + protected static String buildReceiptHandle(String topic, long popTime, long invisibleTime) { + return ExtraInfoUtil.buildExtraInfo( + RANDOM.nextInt(Integer.MAX_VALUE), + popTime, + invisibleTime, + 0, + topic, + "brokerName", + RANDOM.nextInt(8), + RANDOM.nextInt(Integer.MAX_VALUE) + ); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java new file mode 100644 index 0000000000..350f5802a3 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/GrpcMessagingApplicationTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2; + +import apache.rocketmq.v2.Address; +import apache.rocketmq.v2.AddressScheme; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Endpoints; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.Resource; +import io.grpc.Context; +import io.grpc.Metadata; +import io.grpc.stub.StreamObserver; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; + +@RunWith(MockitoJUnitRunner.class) +public class GrpcMessagingApplicationTest extends InitConfigAndLoggerTest { + protected static final String REMOTE_ADDR = "192.168.0.1:8080"; + protected static final String LOCAL_ADDR = "127.0.0.1:8080"; + protected static final String CLIENT_ID = "client-id" + UUID.randomUUID(); + protected static final String JAVA = "JAVA"; + @Mock + StreamObserver queryRouteResponseStreamObserver; + @Mock + GrpcMessingActivity grpcMessingActivity; + GrpcMessagingApplication grpcMessagingApplication; + + private static final String TOPIC = "topic"; + private static Endpoints grpcEndpoints = Endpoints.newBuilder() + .setScheme(AddressScheme.IPv4) + .addAddresses(Address.newBuilder().setHost("127.0.0.1").setPort(8080).build()) + .addAddresses(Address.newBuilder().setHost("127.0.0.2").setPort(8080).build()) + .build(); + + @Before + public void setUp() throws Throwable { + super.before(); + grpcMessagingApplication = new GrpcMessagingApplication(grpcMessingActivity); + } + + @Test + public void testQueryRoute() { + Metadata metadata = new Metadata(); + metadata.put(InterceptorConstants.CLIENT_ID, CLIENT_ID); + metadata.put(InterceptorConstants.LANGUAGE, JAVA); + metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR); + metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR); + Context.current() + .withValue(InterceptorConstants.METADATA, metadata) + .attach(); + + CompletableFuture future = new CompletableFuture<>(); + QueryRouteRequest request = QueryRouteRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .build(); + Mockito.when(grpcMessingActivity.queryRoute(Mockito.any(ProxyContext.class), Mockito.eq(request))) + .thenReturn(future); + QueryRouteResponse response = QueryRouteResponse.newBuilder() + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .addMessageQueues(MessageQueue.getDefaultInstance()) + .build(); + grpcMessagingApplication.queryRoute(request, queryRouteResponseStreamObserver); + future.complete(response); + await().untilAsserted(() -> { + Mockito.verify(queryRouteResponseStreamObserver, Mockito.times(1)).onNext(Mockito.same(response)); + }); + } + + @Test + public void testQueryRouteWithBadClientID() { + Metadata metadata = new Metadata(); + metadata.put(InterceptorConstants.LANGUAGE, JAVA); + metadata.put(InterceptorConstants.REMOTE_ADDRESS, REMOTE_ADDR); + metadata.put(InterceptorConstants.LOCAL_ADDRESS, LOCAL_ADDR); + Context.current() + .withValue(InterceptorConstants.METADATA, metadata) + .attach(); + + QueryRouteRequest request = QueryRouteRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .build(); + grpcMessagingApplication.queryRoute(request, queryRouteResponseStreamObserver); + + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(QueryRouteResponse.class); + await().untilAsserted(() -> { + Mockito.verify(queryRouteResponseStreamObserver, Mockito.times(1)).onNext(responseArgumentCaptor.capture()); + }); + + assertEquals(Code.CLIENT_ID_REQUIRED, responseArgumentCaptor.getValue().getStatus().getCode()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivityTest.java new file mode 100644 index 0000000000..89299cf4c9 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/client/ClientActivityTest.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.client; + +import apache.rocketmq.v2.ClientType; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.HeartbeatResponse; +import apache.rocketmq.v2.NotifyClientTerminationRequest; +import apache.rocketmq.v2.NotifyClientTerminationResponse; +import apache.rocketmq.v2.Publishing; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import apache.rocketmq.v2.SubscriptionEntry; +import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.ThreadStackTrace; +import apache.rocketmq.v2.VerifyMessageResult; +import io.grpc.stub.StreamObserver; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.CMResult; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; +import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcClientChannel; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayResult; +import org.apache.rocketmq.remoting.protocol.LanguageCode; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ClientActivityTest extends BaseActivityTest { + + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; + + private ClientActivity clientActivity; + @Mock + private GrpcChannelManager grpcChannelManagerMock; + @Mock + private CompletableFuture> runningInfoFutureMock; + @Captor + ArgumentCaptor> runningInfoArgumentCaptor; + @Mock + private CompletableFuture> resultFutureMock; + @Captor + ArgumentCaptor> resultArgumentCaptor; + + @Before + public void before() throws Throwable { + super.before(); + this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManager); + } + + protected TelemetryCommand sendProducerTelemetry(ProxyContext context) throws Throwable { + return this.sendClientTelemetry( + context, + Settings.newBuilder() + .setClientType(ClientType.PRODUCER) + .setPublishing(Publishing.newBuilder() + .addTopics(Resource.newBuilder().setName(TOPIC).build()) + .build()) + .build()).get(); + } + + protected HeartbeatResponse sendProducerHeartbeat(ProxyContext context) throws Throwable { + return this.clientActivity.heartbeat(context, HeartbeatRequest.newBuilder() + .setClientType(ClientType.PRODUCER) + .build()).get(); + } + + @Test + public void testProducerHeartbeat() throws Throwable { + ProxyContext context = createContext(); + + this.sendProducerTelemetry(context); + + ArgumentCaptor registerProducerGroupArgumentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doNothing().when(this.messagingProcessor).registerProducer(any(), + registerProducerGroupArgumentCaptor.capture(), + channelInfoArgumentCaptor.capture()); + + ArgumentCaptor txProducerGroupArgumentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor txProducerTopicArgumentCaptor = ArgumentCaptor.forClass(String.class); + doNothing().when(this.messagingProcessor).addTransactionSubscription(any(), + txProducerGroupArgumentCaptor.capture(), + txProducerTopicArgumentCaptor.capture() + ); + + HeartbeatResponse response = this.sendProducerHeartbeat(context); + + assertEquals(Code.OK, response.getStatus().getCode()); + + assertEquals(Lists.newArrayList(TOPIC), registerProducerGroupArgumentCaptor.getAllValues()); + ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue(); + assertClientChannelInfo(clientChannelInfo, TOPIC); + + assertEquals(Lists.newArrayList(TOPIC), txProducerGroupArgumentCaptor.getAllValues()); + assertEquals(Lists.newArrayList(TOPIC), txProducerTopicArgumentCaptor.getAllValues()); + } + + protected TelemetryCommand sendConsumerTelemetry(ProxyContext context) throws Throwable { + return this.sendClientTelemetry( + context, + Settings.newBuilder() + .setClientType(ClientType.PUSH_CONSUMER) + .setSubscription(Subscription.newBuilder() + .addSubscriptions(SubscriptionEntry.newBuilder() + .setExpression(FilterExpression.newBuilder() + .setExpression("tag") + .setType(FilterType.TAG) + .build()) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .build()) + .build()) + .build()).get(); + } + + protected HeartbeatResponse sendConsumerHeartbeat(ProxyContext context) throws Throwable { + return this.clientActivity.heartbeat(context, HeartbeatRequest.newBuilder() + .setClientType(ClientType.PUSH_CONSUMER) + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .build()).get(); + } + + @Test + public void testConsumerHeartbeat() throws Throwable { + ProxyContext context = createContext(); + this.sendConsumerTelemetry(context); + + ArgumentCaptor> subscriptionDatasArgumentCaptor = ArgumentCaptor.forClass(Set.class); + ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doNothing().when(this.messagingProcessor).registerConsumer(any(), + anyString(), + channelInfoArgumentCaptor.capture(), + any(), + any(), + any(), + subscriptionDatasArgumentCaptor.capture(), + anyBoolean() + ); + + HeartbeatResponse response = this.sendConsumerHeartbeat(context); + assertEquals(Code.OK, response.getStatus().getCode()); + + ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue(); + assertClientChannelInfo(clientChannelInfo, CONSUMER_GROUP); + + SubscriptionData data = subscriptionDatasArgumentCaptor.getValue().stream().findAny().get(); + assertEquals("TAG", data.getExpressionType()); + assertEquals("tag", data.getSubString()); + } + + protected void assertClientChannelInfo(ClientChannelInfo clientChannelInfo, String group) { + assertEquals(LanguageCode.JAVA, clientChannelInfo.getLanguage()); + assertEquals(CLIENT_ID, clientChannelInfo.getClientId()); + assertTrue(clientChannelInfo.getChannel() instanceof GrpcClientChannel); + GrpcClientChannel channel = (GrpcClientChannel) clientChannelInfo.getChannel(); + assertEquals(REMOTE_ADDR, channel.getRemoteAddress()); + assertEquals(LOCAL_ADDR, channel.getLocalAddress()); + assertEquals(group, channel.getGroup()); + } + + @Test + public void testProducerNotifyClientTermination() throws Throwable { + ProxyContext context = createContext(); + + when(this.grpcClientSettingsManager.removeAndGetClientSettings(any())).thenReturn(Settings.newBuilder() + .setClientType(ClientType.PRODUCER) + .setPublishing(Publishing.newBuilder() + .addTopics(Resource.newBuilder().setName(TOPIC).build()) + .build()) + .build()); + ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doNothing().when(this.messagingProcessor).unRegisterProducer(any(), anyString(), channelInfoArgumentCaptor.capture()); + + this.sendProducerTelemetry(context); + this.sendProducerHeartbeat(context); + + NotifyClientTerminationResponse response = this.clientActivity.notifyClientTermination( + context, + NotifyClientTerminationRequest.newBuilder() + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue(); + assertClientChannelInfo(clientChannelInfo, TOPIC); + } + + @Test + public void testConsumerNotifyClientTermination() throws Throwable { + ProxyContext context = createContext(); + + when(this.grpcClientSettingsManager.removeAndGetClientSettings(any())).thenReturn(Settings.newBuilder() + .setClientType(ClientType.PUSH_CONSUMER) + .build()); + ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); + doNothing().when(this.messagingProcessor).unRegisterConsumer(any(), anyString(), channelInfoArgumentCaptor.capture()); + + this.sendConsumerTelemetry(context); + this.sendConsumerHeartbeat(context); + + NotifyClientTerminationResponse response = this.clientActivity.notifyClientTermination( + context, + NotifyClientTerminationRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + ClientChannelInfo clientChannelInfo = channelInfoArgumentCaptor.getValue(); + assertClientChannelInfo(clientChannelInfo, CONSUMER_GROUP); + } + + @Test + public void testReportThreadStackTrace() { + this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManagerMock); + String jstack = "jstack"; + String nonce = "123"; + when(grpcChannelManagerMock.getAndRemoveResponseFuture(anyString())).thenReturn((CompletableFuture) runningInfoFutureMock); + ProxyContext context = createContext(); + StreamObserver streamObserver = clientActivity.telemetry(context, new StreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + } + + @Override + public void onError(Throwable t) { + } + + @Override + public void onCompleted() { + } + }); + streamObserver.onNext(TelemetryCommand.newBuilder() + .setThreadStackTrace(ThreadStackTrace.newBuilder() + .setThreadStackTrace(jstack) + .setNonce(nonce) + .build()) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + verify(runningInfoFutureMock, times(1)).complete(runningInfoArgumentCaptor.capture()); + ProxyRelayResult result = runningInfoArgumentCaptor.getValue(); + assertThat(result.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(result.getResult().getJstack()).isEqualTo(jstack); + } + + @Test + public void testReportVerifyMessageResult() { + this.clientActivity = new ClientActivity(this.messagingProcessor, this.grpcClientSettingsManager, grpcChannelManagerMock); + String nonce = "123"; + when(grpcChannelManagerMock.getAndRemoveResponseFuture(anyString())).thenReturn((CompletableFuture) resultFutureMock); + ProxyContext context = createContext(); + StreamObserver streamObserver = clientActivity.telemetry(context, new StreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + } + + @Override + public void onError(Throwable t) { + } + + @Override + public void onCompleted() { + } + }); + streamObserver.onNext(TelemetryCommand.newBuilder() + .setVerifyMessageResult(VerifyMessageResult.newBuilder() + .setNonce(nonce) + .build()) + .setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())) + .build()); + verify(resultFutureMock, times(1)).complete(resultArgumentCaptor.capture()); + ProxyRelayResult result = resultArgumentCaptor.getValue(); + assertThat(result.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(result.getResult().getConsumeResult()).isEqualTo(CMResult.CR_SUCCESS); + } + + protected CompletableFuture sendClientTelemetry(ProxyContext ctx, Settings settings) { + when(grpcClientSettingsManager.getClientSettings(any())).thenReturn(settings); + + CompletableFuture future = new CompletableFuture<>(); + StreamObserver responseObserver = new StreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + future.complete(value); + } + + @Override + public void onError(Throwable t) { + + } + + @Override + public void onCompleted() { + + } + }; + StreamObserver requestObserver = this.clientActivity.telemetry( + ctx, + responseObserver + ); + requestObserver.onNext(TelemetryCommand.newBuilder() + .setSettings(settings) + .build()); + return future; + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManagerTest.java new file mode 100644 index 0000000000..05ed615119 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcClientSettingsManagerTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.CustomizedBackoff; +import apache.rocketmq.v2.ExponentialBackoff; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.RetryPolicy; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import com.google.protobuf.util.Durations; +import org.apache.rocketmq.common.subscription.CustomizedRetryPolicy; +import org.apache.rocketmq.common.subscription.ExponentialRetryPolicy; +import org.apache.rocketmq.common.subscription.GroupRetryPolicyType; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.common.ContextVariable; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +public class GrpcClientSettingsManagerTest extends BaseActivityTest { + private GrpcClientSettingsManager grpcClientSettingsManager; + + @Before + public void before() throws Throwable { + super.before(); + this.grpcClientSettingsManager = new GrpcClientSettingsManager(this.messagingProcessor); + } + + @Test + public void testGetSubscriptionData() { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + when(this.messagingProcessor.getSubscriptionGroupConfig(any(), any())) + .thenReturn(subscriptionGroupConfig); + + this.grpcClientSettingsManager.updateClientSettings(CLIENT_ID, Settings.newBuilder() + .setSubscription(Subscription.newBuilder() + .setGroup(Resource.newBuilder().setName("group").build()) + .build()) + .build()); + + ProxyContext context = ProxyContext.create().withVal(ContextVariable.CLIENT_ID, CLIENT_ID); + + Settings settings = this.grpcClientSettingsManager.getClientSettings(context); + assertEquals(settings.getBackoffPolicy(), this.grpcClientSettingsManager.createDefaultConsumerSettingsBuilder().build().getBackoffPolicy()); + + subscriptionGroupConfig.setRetryMaxTimes(3); + subscriptionGroupConfig.getGroupRetryPolicy().setType(GroupRetryPolicyType.CUSTOMIZED); + subscriptionGroupConfig.getGroupRetryPolicy().setCustomizedRetryPolicy(new CustomizedRetryPolicy(new long[] {1000})); + settings = this.grpcClientSettingsManager.getClientSettings(context); + assertEquals(RetryPolicy.newBuilder() + .setMaxAttempts(4) + .setCustomizedBackoff(CustomizedBackoff.newBuilder() + .addNext(Durations.fromSeconds(1)) + .build()) + .build(), settings.getBackoffPolicy()); + + subscriptionGroupConfig.setRetryMaxTimes(10); + subscriptionGroupConfig.getGroupRetryPolicy().setType(GroupRetryPolicyType.EXPONENTIAL); + subscriptionGroupConfig.getGroupRetryPolicy().setExponentialRetryPolicy(new ExponentialRetryPolicy(1000, 2000, 3)); + settings = this.grpcClientSettingsManager.getClientSettings(context); + assertEquals(RetryPolicy.newBuilder() + .setMaxAttempts(11) + .setExponentialBackoff(ExponentialBackoff.newBuilder() + .setMax(Durations.fromSeconds(2)) + .setInitial(Durations.fromSeconds(1)) + .setMultiplier(3) + .build()) + .build(), settings.getBackoffPolicy()); + + Settings settings1 = this.grpcClientSettingsManager.removeAndGetClientSettings(context); + assertEquals(settings, settings1); + + assertNull(this.grpcClientSettingsManager.getClientSettings(context)); + assertNull(this.grpcClientSettingsManager.removeAndGetClientSettings(context)); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverterTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverterTest.java new file mode 100644 index 0000000000..bc9b8a60b4 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/common/GrpcConverterTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.common; + +import apache.rocketmq.v2.MessageQueue; +import org.apache.rocketmq.common.message.MessageExt; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class GrpcConverterTest { + @Test + public void testBuildMessageQueue() { + String topic = "topic"; + String brokerName = "brokerName"; + int queueId = 1; + MessageExt messageExt = new MessageExt(); + messageExt.setQueueId(queueId); + messageExt.setTopic(topic); + + MessageQueue messageQueue = GrpcConverter.getInstance().buildMessageQueue(messageExt, brokerName); + assertThat(messageQueue.getTopic().getName()).isEqualTo(topic); + assertThat(messageQueue.getBroker().getName()).isEqualTo(brokerName); + assertThat(messageQueue.getId()).isEqualTo(queueId); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java new file mode 100644 index 0000000000..4df834bb65 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.AckMessageEntry; +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Resource; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +public class AckMessageActivityTest extends BaseActivityTest { + + private AckMessageActivity ackMessageActivity; + + private static final String TOPIC = "topic"; + private static final String GROUP = "group"; + + @Before + public void before() throws Throwable { + super.before(); + this.ackMessageActivity = new AckMessageActivity(messagingProcessor, receiptHandleProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testAckMessage() throws Throwable { + when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) + .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); + + AckResult msg2AckResult = new AckResult(); + msg2AckResult.setStatus(AckStatus.OK); + when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); + + AckResult msg3AckResult = new AckResult(); + msg3AckResult.setStatus(AckStatus.NO_EXIST); + when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); + + AckMessageResponse response = this.ackMessageActivity.ackMessage( + createContext(), + AckMessageRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(GROUP).build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId("msg1") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId("msg2") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .addEntries(AckMessageEntry.newBuilder() + .setMessageId("msg3") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build()) + .build() + ).get(); + + assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); + assertEquals(3, response.getEntriesCount()); + assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); + assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); + assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivityTest.java new file mode 100644 index 0000000000..44f879f839 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ChangeInvisibleDurationActivityTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Resource; +import com.google.protobuf.util.Durations; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +public class ChangeInvisibleDurationActivityTest extends BaseActivityTest { + + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private ChangeInvisibleDurationActivity changeInvisibleDurationActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.changeInvisibleDurationActivity = new ChangeInvisibleDurationActivity(messagingProcessor, receiptHandleProcessor, + grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testChangeInvisibleDurationActivity() throws Throwable { + String newHandle = "newHandle"; + ArgumentCaptor invisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class); + AckResult ackResult = new AckResult(); + ackResult.setExtraInfo(newHandle); + ackResult.setStatus(AckStatus.OK); + when(this.messagingProcessor.changeInvisibleTime( + any(), any(), anyString(), anyString(), anyString(), invisibleTimeArgumentCaptor.capture() + )).thenReturn(CompletableFuture.completedFuture(ackResult)); + + ChangeInvisibleDurationResponse response = this.changeInvisibleDurationActivity.changeInvisibleDuration( + createContext(), + ChangeInvisibleDurationRequest.newBuilder() + .setInvisibleDuration(Durations.fromSeconds(3)) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageId("msgId") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(TimeUnit.SECONDS.toMillis(3), invisibleTimeArgumentCaptor.getValue().longValue()); + assertEquals(newHandle, response.getReceiptHandle()); + } + + @Test + public void testChangeInvisibleDurationActivityFailed() throws Throwable { + ArgumentCaptor invisibleTimeArgumentCaptor = ArgumentCaptor.forClass(Long.class); + AckResult ackResult = new AckResult(); + ackResult.setStatus(AckStatus.NO_EXIST); + when(this.messagingProcessor.changeInvisibleTime( + any(), any(), anyString(), anyString(), anyString(), invisibleTimeArgumentCaptor.capture() + )).thenReturn(CompletableFuture.completedFuture(ackResult)); + + ChangeInvisibleDurationResponse response = this.changeInvisibleDurationActivity.changeInvisibleDuration( + createContext(), + ChangeInvisibleDurationRequest.newBuilder() + .setInvisibleDuration(Durations.fromSeconds(3)) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageId("msgId") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build() + ).get(); + + assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); + assertEquals(TimeUnit.SECONDS.toMillis(3), invisibleTimeArgumentCaptor.getValue().longValue()); + } + + @Test + public void testChangeInvisibleDurationInvisibleTimeTooSmall() throws Throwable { + try { + this.changeInvisibleDurationActivity.changeInvisibleDuration( + createContext(), + ChangeInvisibleDurationRequest.newBuilder() + .setInvisibleDuration(Durations.fromSeconds(-1)) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageId("msgId") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build() + ).get(); + } catch (ExecutionException executionException) { + GrpcProxyException exception = (GrpcProxyException) executionException.getCause(); + assertEquals(Code.ILLEGAL_INVISIBLE_TIME, exception.getCode()); + } + } + + @Test + public void testChangeInvisibleDurationInvisibleTimeTooLarge() throws Throwable { + try { + this.changeInvisibleDurationActivity.changeInvisibleDuration( + createContext(), + ChangeInvisibleDurationRequest.newBuilder() + .setInvisibleDuration(Durations.fromDays(7)) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageId("msgId") + .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) + .build() + ).get(); + } catch (ExecutionException executionException) { + GrpcProxyException exception = (GrpcProxyException) executionException.getCause(); + assertEquals(Code.ILLEGAL_INVISIBLE_TIME, exception.getCode()); + } + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java new file mode 100644 index 0000000000..2df5480a73 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.Settings; +import com.google.protobuf.util.Durations; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ReceiveMessageActivityTest extends BaseActivityTest { + + protected static final String BROKER_NAME = "broker"; + protected static final String CLUSTER_NAME = "cluster"; + protected static final String BROKER_ADDR = "127.0.0.1:10911"; + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private ReceiveMessageActivity receiveMessageActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.receiveMessageActivity = new ReceiveMessageActivity(messagingProcessor, receiptHandleProcessor, + grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testReceiveMessageIllegalFilter() { + StreamObserver receiveStreamObserver = mock(ServerCallStreamObserver.class); + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture()); + + when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType()); + + this.receiveMessageActivity.receiveMessage( + createContext(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setAutoRenew(true) + .setFilterExpression(FilterExpression.newBuilder() + .setType(FilterType.SQL) + .setExpression("") + .build()) + .build(), + receiveStreamObserver + ); + + assertEquals(Code.ILLEGAL_FILTER_EXPRESSION, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues())); + } + + @Test + public void testReceiveMessageIllegalInvisibleTimeTooSmall() { + StreamObserver receiveStreamObserver = mock(ServerCallStreamObserver.class); + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture()); + + when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType()); + + this.receiveMessageActivity.receiveMessage( + createContext(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setAutoRenew(false) + .setInvisibleDuration(Durations.fromSeconds(0)) + .build(), + receiveStreamObserver + ); + + assertEquals(Code.ILLEGAL_INVISIBLE_TIME, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues())); + } + + @Test + public void testReceiveMessageIllegalInvisibleTimeTooLarge() { + StreamObserver receiveStreamObserver = mock(ServerCallStreamObserver.class); + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture()); + + when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType()); + + this.receiveMessageActivity.receiveMessage( + createContext(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setAutoRenew(false) + .setInvisibleDuration(Durations.fromDays(7)) + .build(), + receiveStreamObserver + ); + + assertEquals(Code.ILLEGAL_INVISIBLE_TIME, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues())); + } + + + @Test + public void testReceiveMessage() { + StreamObserver receiveStreamObserver = mock(ServerCallStreamObserver.class); + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + doNothing().when(receiveStreamObserver).onNext(responseArgumentCaptor.capture()); + + when(this.grpcClientSettingsManager.getClientSettings(any())).thenReturn(Settings.newBuilder().getDefaultInstanceForType()); + + PopResult popResult = new PopResult(PopStatus.NO_NEW_MSG, new ArrayList<>()); + when(this.messagingProcessor.popMessage( + any(), + any(), + anyString(), + anyString(), + anyInt(), + anyLong(), + anyLong(), + anyInt(), + any(), + anyBoolean(), + any(), + anyLong())).thenReturn(CompletableFuture.completedFuture(popResult)); + + this.receiveMessageActivity.receiveMessage( + createContext(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setAutoRenew(true) + .setFilterExpression(FilterExpression.newBuilder() + .setType(FilterType.TAG) + .setExpression("*") + .build()) + .build(), + receiveStreamObserver + ); + assertEquals(Code.MESSAGE_NOT_FOUND, getResponseCodeFromReceiveMessageResponseList(responseArgumentCaptor.getAllValues())); + } + + private Code getResponseCodeFromReceiveMessageResponseList(List responseList) { + for (ReceiveMessageResponse response : responseList) { + if (response.hasStatus()) { + return response.getStatus().getCode(); + } + } + return null; + } + + @Test + public void testReceiveMessageQueueSelector() { + TopicRouteData topicRouteData = new TopicRouteData(); + List queueDatas = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(BROKER_NAME + i); + queueData.setReadQueueNums(1); + queueData.setPerm(PermName.PERM_READ); + queueDatas.add(queueData); + } + topicRouteData.setQueueDatas(queueDatas); + + List brokerDatas = new ArrayList<>(); + for (int i = 0; i < 2; i++) { + BrokerData brokerData = new BrokerData(); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME + i); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); + brokerData.setBrokerAddrs(brokerAddrs); + brokerDatas.add(brokerData); + } + topicRouteData.setBrokerDatas(brokerDatas); + + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(""); + + AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); + AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); + AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView); + + assertEquals(firstSelect, thirdSelect); + assertNotEquals(firstSelect, secondSelect); + + for (int i = 0; i < 2; i++) { + ReceiveMessageActivity.ReceiveMessageQueueSelector selectorBrokerName = + new ReceiveMessageActivity.ReceiveMessageQueueSelector(BROKER_NAME + i); + assertEquals(BROKER_NAME + i, selectorBrokerName.select(ProxyContext.create(), messageQueueView).getBrokerName()); + } + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriterTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriterTest.java new file mode 100644 index 0000000000..8086b37b36 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageResponseStreamWriterTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.consumer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.FilterExpression; +import apache.rocketmq.v2.FilterType; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.Resource; +import io.grpc.stub.StreamObserver; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class ReceiveMessageResponseStreamWriterTest extends BaseActivityTest { + + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private ReceiveMessageResponseStreamWriter writer; + private StreamObserver streamObserver; + + @Before + public void before() throws Throwable { + super.before(); + this.streamObserver = mock(StreamObserver.class); + this.writer = new ReceiveMessageResponseStreamWriter(this.messagingProcessor, this.streamObserver); + } + + @Test + public void testWriteMessage() { + ArgumentCaptor changeInvisibleTimeMsgIdCaptor = ArgumentCaptor.forClass(String.class); + doReturn(CompletableFuture.completedFuture(mock(AckResult.class))).when(this.messagingProcessor) + .changeInvisibleTime(any(), any(), changeInvisibleTimeMsgIdCaptor.capture(), anyString(), anyString(), anyLong()); + + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + AtomicInteger onNextCallNum = new AtomicInteger(0); + doAnswer(mock -> { + if (onNextCallNum.incrementAndGet() > 2) { + throw new RuntimeException(); + } + return null; + }).when(streamObserver).onNext(responseArgumentCaptor.capture()); + + List messageExtList = new ArrayList<>(); + messageExtList.add(createMessageExt(TOPIC, "tag")); + messageExtList.add(createMessageExt(TOPIC, "tag")); + PopResult popResult = new PopResult(PopStatus.FOUND, messageExtList); + writer.writeAndComplete( + ProxyContext.create(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setFilterExpression(FilterExpression.newBuilder() + .setType(FilterType.TAG) + .setExpression("*") + .build()) + .build(), + popResult + ); + + verify(streamObserver, times(1)).onCompleted(); + verify(streamObserver, times(4)).onNext(any()); + verify(this.messagingProcessor, times(1)) + .changeInvisibleTime(any(), any(), anyString(), anyString(), anyString(), anyLong()); + + assertTrue(responseArgumentCaptor.getAllValues().get(0).hasStatus()); + assertEquals(Code.OK, responseArgumentCaptor.getAllValues().get(0).getStatus().getCode()); + assertTrue(responseArgumentCaptor.getAllValues().get(1).hasMessage()); + assertEquals(messageExtList.get(0).getMsgId(), responseArgumentCaptor.getAllValues().get(1).getMessage().getSystemProperties().getMessageId()); + + assertEquals(messageExtList.get(1).getMsgId(), changeInvisibleTimeMsgIdCaptor.getValue()); + } + + @Test + public void testPollingFull() { + ArgumentCaptor responseArgumentCaptor = ArgumentCaptor.forClass(ReceiveMessageResponse.class); + doNothing().when(streamObserver).onNext(responseArgumentCaptor.capture()); + + PopResult popResult = new PopResult(PopStatus.POLLING_FULL, new ArrayList<>()); + writer.writeAndComplete( + ProxyContext.create(), + ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder().setName(CONSUMER_GROUP).build()) + .setMessageQueue(MessageQueue.newBuilder().setTopic(Resource.newBuilder().setName(TOPIC).build()).build()) + .setFilterExpression(FilterExpression.newBuilder() + .setType(FilterType.TAG) + .setExpression("*") + .build()) + .build(), + popResult + ); + + ReceiveMessageResponse response = responseArgumentCaptor.getAllValues().stream().filter(ReceiveMessageResponse::hasStatus) + .findFirst().get(); + assertEquals(Code.TOO_MANY_REQUESTS, response.getStatus().getCode()); + } + + private static MessageExt createMessageExt(String topic, String tags) { + String msgId = MessageClientIDSetter.createUniqID(); + + MessageExt messageExt = new MessageExt(); + messageExt.setTopic(topic); + messageExt.setTags(tags); + messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + messageExt.setMsgId(msgId); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_UNIQ_CLIENT_MESSAGE_ID_KEYIDX, msgId); + messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, + ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), 3000, + RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); + return messageExt; + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivityTest.java new file mode 100644 index 0000000000..3e2a0b4200 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/ForwardMessageToDLQActivityTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest; +import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueResponse; +import apache.rocketmq.v2.Resource; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +public class ForwardMessageToDLQActivityTest extends BaseActivityTest { + + private ForwardMessageToDLQActivity forwardMessageToDLQActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.forwardMessageToDLQActivity = new ForwardMessageToDLQActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testForwardMessageToDeadLetterQueue() throws Throwable { + when(this.messagingProcessor.forwardMessageToDeadLetterQueue(any(), any(), anyString(), anyString(), anyString())) + .thenReturn(CompletableFuture.completedFuture(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""))); + + ForwardMessageToDeadLetterQueueResponse response = this.forwardMessageToDLQActivity.forwardMessageToDeadLetterQueue( + createContext(), + ForwardMessageToDeadLetterQueueRequest.newBuilder() + .setTopic(Resource.newBuilder().setName("topic").build()) + .setGroup(Resource.newBuilder().setName("group").build()) + .setMessageId(MessageClientIDSetter.createUniqID()) + .setReceiptHandle(buildReceiptHandle("topic", System.currentTimeMillis(), 3000)) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java new file mode 100644 index 0000000000..7776d387e5 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java @@ -0,0 +1,793 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.producer; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Encoding; +import apache.rocketmq.v2.Message; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.SystemProperties; +import com.google.protobuf.ByteString; +import com.google.protobuf.util.Durations; +import com.google.protobuf.util.Timestamps; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +public class SendMessageActivityTest extends BaseActivityTest { + + protected static final String BROKER_NAME = "broker"; + protected static final String CLUSTER_NAME = "cluster"; + protected static final String BROKER_ADDR = "127.0.0.1:10911"; + private static final String TOPIC = "topic"; + private static final String CONSUMER_GROUP = "consumerGroup"; + + private SendMessageActivity sendMessageActivity; + + @Before + public void before() throws Throwable { + super.before(); + this.sendMessageActivity = new SendMessageActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void sendMessage() throws Exception { + String msgId = MessageClientIDSetter.createUniqID(); + + SendResult sendResult = new SendResult(); + sendResult.setSendStatus(SendStatus.SEND_OK); + sendResult.setMsgId(msgId); + when(this.messagingProcessor.sendMessage(any(), any(), anyString(), anyInt(), any())) + .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult))); + + SendMessageResponse response = this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(msgId) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build()) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(msgId, response.getEntries(0).getMessageId()); + } + + @Test + public void testConvertToSendMessageResponse() { + { + SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse( + ProxyContext.create(), + SendMessageRequest.newBuilder().build(), + Lists.newArrayList(new SendResult(SendStatus.FLUSH_DISK_TIMEOUT, null, null, null, 0)) + ); + assertEquals(Code.MASTER_PERSISTENCE_TIMEOUT, response.getStatus().getCode()); + assertEquals(Code.MASTER_PERSISTENCE_TIMEOUT, response.getEntries(0).getStatus().getCode()); + } + + { + SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse( + ProxyContext.create(), + SendMessageRequest.newBuilder().build(), + Lists.newArrayList(new SendResult(SendStatus.FLUSH_SLAVE_TIMEOUT, null, null, null, 0)) + ); + assertEquals(Code.SLAVE_PERSISTENCE_TIMEOUT, response.getStatus().getCode()); + assertEquals(Code.SLAVE_PERSISTENCE_TIMEOUT, response.getEntries(0).getStatus().getCode()); + } + + { + SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse( + ProxyContext.create(), + SendMessageRequest.newBuilder().build(), + Lists.newArrayList(new SendResult(SendStatus.SLAVE_NOT_AVAILABLE, null, null, null, 0)) + ); + assertEquals(Code.HA_NOT_AVAILABLE, response.getStatus().getCode()); + assertEquals(Code.HA_NOT_AVAILABLE, response.getEntries(0).getStatus().getCode()); + } + + { + SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse( + ProxyContext.create(), + SendMessageRequest.newBuilder().build(), + Lists.newArrayList(new SendResult(SendStatus.SEND_OK, null, null, null, 0)) + ); + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(Code.OK, response.getEntries(0).getStatus().getCode()); + } + + { + SendMessageResponse response = this.sendMessageActivity.convertToSendMessageResponse( + ProxyContext.create(), + SendMessageRequest.newBuilder().build(), + Lists.newArrayList( + new SendResult(SendStatus.SEND_OK, null, null, null, 0), + new SendResult(SendStatus.SLAVE_NOT_AVAILABLE, null, null, null, 0) + ) + ); + assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); + } + } + + @Test(expected = GrpcProxyException.class) + public void testBuildErrorMessage() { + this.sendMessageActivity.buildMessage(null, + Lists.newArrayList( + Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(MessageClientIDSetter.createUniqID()) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build(), + Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC + 2) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(MessageClientIDSetter.createUniqID()) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build() + ), + Resource.newBuilder().setName(TOPIC).build()); + } + + @Test + public void testBuildMessage() { + long deliveryTime = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5); + String msgId = MessageClientIDSetter.createUniqID(); + + org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(null, + Lists.newArrayList( + Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(msgId) + .setQueueId(0) + .setMessageType(MessageType.DELAY) + .setDeliveryTimestamp(Timestamps.fromMillis(deliveryTime)) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build() + ), + Resource.newBuilder().setName(TOPIC).build()).get(0); + + assertEquals(MessageClientIDSetter.getUniqID(messageExt), msgId); + assertEquals(String.valueOf(2), messageExt.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL)); + } + + @Test + public void testTxMessage() { + String msgId = MessageClientIDSetter.createUniqID(); + + Message message = Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(msgId) + .setQueueId(0) + .setMessageType(MessageType.TRANSACTION) + .setOrphanedTransactionRecoveryDuration(Durations.fromSeconds(30)) + .setBodyEncoding(Encoding.GZIP) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build(); + org.apache.rocketmq.common.message.Message messageExt = this.sendMessageActivity.buildMessage(null, + Lists.newArrayList( + message + ), + Resource.newBuilder().setName(TOPIC).build()).get(0); + + assertEquals(MessageClientIDSetter.getUniqID(messageExt), msgId); + assertEquals(MessageSysFlag.TRANSACTION_PREPARED_TYPE | MessageSysFlag.COMPRESSED_FLAG, sendMessageActivity.buildSysFlag(message)); + } + + @Test + public void testSendOrderMessageQueueSelector() { + TopicRouteData topicRouteData = new TopicRouteData(); + QueueData queueData = new QueueData(); + BrokerData brokerData = new BrokerData(); + queueData.setBrokerName(BROKER_NAME); + queueData.setWriteQueueNums(8); + queueData.setPerm(PermName.PERM_WRITE); + topicRouteData.setQueueDatas(Lists.newArrayList(queueData)); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setSystemProperties(SystemProperties.newBuilder() + .setMessageGroup(String.valueOf(1)) + .build()) + .build()) + .build() + ); + + SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setSystemProperties(SystemProperties.newBuilder() + .setMessageGroup(String.valueOf(1)) + .build()) + .build()) + .build() + ); + + SendMessageActivity.SendMessageQueueSelector selector3 = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setSystemProperties(SystemProperties.newBuilder() + .setMessageGroup(String.valueOf(2)) + .build()) + .build()) + .build() + ); + + assertEquals(selector1.select(ProxyContext.create(), messageQueueView), selector2.select(ProxyContext.create(), messageQueueView)); + assertNotEquals(selector1.select(ProxyContext.create(), messageQueueView), selector3.select(ProxyContext.create(), messageQueueView)); + } + + @Test + public void testSendNormalMessageQueueSelector() { + TopicRouteData topicRouteData = new TopicRouteData(); + QueueData queueData = new QueueData(); + BrokerData brokerData = new BrokerData(); + queueData.setBrokerName(BROKER_NAME); + queueData.setWriteQueueNums(2); + queueData.setPerm(PermName.PERM_WRITE); + topicRouteData.setQueueDatas(Lists.newArrayList(queueData)); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder().build()) + .build() + ); + + AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); + AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); + AddressableMessageQueue thirdSelect = selector.select(ProxyContext.create(), messageQueueView); + + assertEquals(firstSelect, thirdSelect); + assertNotEquals(firstSelect, secondSelect); + } + + @Test + public void testParameterValidate() { + // too large message body + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[4 * 1024 * 1024 + 1])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.MESSAGE_BODY_TOO_LARGE, e.getCode()); + throw e; + } + }); + + // black tag + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setTag(" ") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode()); + throw e; + } + }); + + // tag with '|' + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setTag("|") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode()); + throw e; + } + }); + + // tag with \t + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setTag("\t") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_TAG, e.getCode()); + throw e; + } + }); + + // blank message key + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .addKeys(" ") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_KEY, e.getCode()); + throw e; + } + }); + + // blank message with \t + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .addKeys("\t") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_KEY, e.getCode()); + throw e; + } + }); + + // blank message group + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageGroup(" ") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode()); + throw e; + } + }); + + // long message group + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageGroup(createStr(ConfigurationManager.getProxyConfig().getMaxMessageGroupSize() + 1)) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode()); + throw e; + } + }); + + // message group with \t + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageGroup("\t") + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_GROUP, e.getCode()); + throw e; + } + }); + + // too large message property + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .putUserProperties("key", createStr(16 * 1024 + 1)) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.MESSAGE_PROPERTIES_TOO_LARGE, e.getCode()); + throw e; + } + }); + + // too large message property + assertThrows(GrpcProxyException.class, () -> { + Map p = new HashMap<>(); + for (int i = 0; i <= ConfigurationManager.getProxyConfig().getUserPropertyMaxNum(); i++) { + p.put(String.valueOf(i), String.valueOf(i)); + } + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .putAllUserProperties(p) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.MESSAGE_PROPERTIES_TOO_LARGE, e.getCode()); + throw e; + } + }); + + // set system properties + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("msgId") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .putUserProperties(MessageConst.PROPERTY_TRACE_SWITCH, "false") + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_PROPERTY_KEY, e.getCode()); + throw e; + } + }); + + // empty message id + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(" ") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_MESSAGE_ID, e.getCode()); + throw e; + } + }); + + // delay time + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("id") + .setDeliveryTimestamp( + Timestamps.fromMillis(System.currentTimeMillis() + Duration.ofDays(1).toMillis() + Duration.ofSeconds(10).toMillis())) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.ILLEGAL_DELIVERY_TIME, e.getCode()); + throw e; + } + }); + + // transactionRecoverySecond + assertThrows(GrpcProxyException.class, () -> { + try { + this.sendMessageActivity.sendMessage( + createContext(), + SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(TOPIC) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId("id") + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .setOrphanedTransactionRecoveryDuration(Durations.fromHours(2)) + .setMessageType(MessageType.TRANSACTION) + .build()) + .setBody(ByteString.copyFrom(new byte[3])) + .build()) + .build() + ).get(); + } catch (ExecutionException t) { + GrpcProxyException e = (GrpcProxyException) t.getCause(); + assertEquals(Code.BAD_REQUEST, e.getCode()); + throw e; + } + }); + } + + private static String createStr(int len) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < len; i++) { + sb.append("a"); + } + return sb.toString(); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java new file mode 100644 index 0000000000..a8c79442b5 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/route/RouteActivityTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.route; + +import apache.rocketmq.v2.Address; +import apache.rocketmq.v2.AddressScheme; +import apache.rocketmq.v2.Broker; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Endpoints; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.Permission; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.Resource; +import com.google.common.net.HostAndPort; +import java.util.ArrayList; +import java.util.List; +import org.apache.rocketmq.client.exception.MQBrokerException; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +public class RouteActivityTest extends BaseActivityTest { + + private RouteActivity routeActivity; + + private static final String CLUSTER = "cluster"; + private static final String TOPIC = "topic"; + private static final String GROUP = "group"; + private static final String BROKER_NAME = "brokerName"; + private static final Broker GRPC_BROKER = Broker.newBuilder().setName(BROKER_NAME).build(); + private static final Resource GRPC_TOPIC = Resource.newBuilder() + .setName(TOPIC) + .build(); + private static final Resource GRPC_GROUP = Resource.newBuilder() + .setName(GROUP) + .build(); + private static Endpoints grpcEndpoints = Endpoints.newBuilder() + .setScheme(AddressScheme.IPv4) + .addAddresses(Address.newBuilder().setHost("127.0.0.1").setPort(8080).build()) + .addAddresses(Address.newBuilder().setHost("127.0.0.2").setPort(8080).build()) + .build(); + private static List addressArrayList = new ArrayList<>(); + + static { + addressArrayList.add(new org.apache.rocketmq.proxy.common.Address( + org.apache.rocketmq.proxy.common.Address.AddressScheme.IPv4, + HostAndPort.fromParts("127.0.0.1", 8080))); + addressArrayList.add(new org.apache.rocketmq.proxy.common.Address( + org.apache.rocketmq.proxy.common.Address.AddressScheme.IPv4, + HostAndPort.fromParts("127.0.0.2", 8080))); + } + + @Before + public void before() throws Throwable { + super.before(); + this.routeActivity = new RouteActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testQueryRoute() throws Throwable { + ConfigurationManager.getProxyConfig().setGrpcServerPort(8080); + ArgumentCaptor> addressListCaptor = ArgumentCaptor.forClass(List.class); + when(this.messagingProcessor.getTopicRouteDataForProxy(any(), addressListCaptor.capture(), anyString())) + .thenReturn(createProxyTopicRouteData(2, 2, 6)); + + QueryRouteResponse response = this.routeActivity.queryRoute( + createContext(), + QueryRouteRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(Resource.newBuilder().setName(TOPIC).build()) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(4, response.getMessageQueuesCount()); + for (MessageQueue messageQueue : response.getMessageQueuesList()) { + assertEquals(grpcEndpoints, messageQueue.getBroker().getEndpoints()); + assertEquals(Permission.READ_WRITE, messageQueue.getPermission()); + } + } + + @Test + public void testQueryRouteTopicExist() throws Throwable { + when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString())) + .thenThrow(new MQBrokerException(ResponseCode.TOPIC_NOT_EXIST, "")); + + try { + this.routeActivity.queryRoute( + createContext(), + QueryRouteRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(GRPC_TOPIC) + .build() + ).get(); + } catch (Throwable t) { + assertEquals(Code.TOPIC_NOT_FOUND, ResponseBuilder.getInstance().buildStatus(t).getCode()); + return; + } + fail(); + } + + @Test + public void testQueryAssignmentWithNoReadPerm() throws Throwable { + when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString())) + .thenReturn(createProxyTopicRouteData(2, 2, PermName.PERM_WRITE)); + + QueryAssignmentResponse response = this.routeActivity.queryAssignment( + createContext(), + QueryAssignmentRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(GRPC_TOPIC) + .setGroup(GRPC_GROUP) + .build() + ).get(); + + assertEquals(Code.FORBIDDEN, response.getStatus().getCode()); + } + + @Test + public void testQueryAssignmentWithNoReadQueue() throws Throwable { + when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString())) + .thenReturn(createProxyTopicRouteData(0, 2, 6)); + + QueryAssignmentResponse response = this.routeActivity.queryAssignment( + createContext(), + QueryAssignmentRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(GRPC_TOPIC) + .setGroup(GRPC_GROUP) + .build() + ).get(); + + assertEquals(Code.FORBIDDEN, response.getStatus().getCode()); + } + + @Test + public void testQueryAssignment() throws Throwable { + when(this.messagingProcessor.getTopicRouteDataForProxy(any(), any(), anyString())) + .thenReturn(createProxyTopicRouteData(2, 2, 6)); + + QueryAssignmentResponse response = this.routeActivity.queryAssignment( + createContext(), + QueryAssignmentRequest.newBuilder() + .setEndpoints(grpcEndpoints) + .setTopic(GRPC_TOPIC) + .setGroup(GRPC_GROUP) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(1, response.getAssignmentsCount()); + assertEquals(grpcEndpoints, response.getAssignments(0).getMessageQueue().getBroker().getEndpoints()); + } + + private static ProxyTopicRouteData createProxyTopicRouteData(int r, int w, int p) { + ProxyTopicRouteData proxyTopicRouteData = new ProxyTopicRouteData(); + proxyTopicRouteData.getQueueDatas().add(createQueueData(r, w, p)); + ProxyTopicRouteData.ProxyBrokerData proxyBrokerData = new ProxyTopicRouteData.ProxyBrokerData(); + proxyBrokerData.setCluster(CLUSTER); + proxyBrokerData.setBrokerName(BROKER_NAME); + proxyBrokerData.getBrokerAddrs().put(0L, addressArrayList); + proxyBrokerData.getBrokerAddrs().put(1L, addressArrayList); + proxyTopicRouteData.getBrokerDatas().add(proxyBrokerData); + return proxyTopicRouteData; + } + + @Test + public void testGenPartitionFromQueueData() throws Exception { + // test queueData with 8 read queues, 8 write queues, and rw permission, expect 8 rw queues. + QueueData queueDataWith8R8WPermRW = createQueueData(8, 8, PermName.PERM_READ | PermName.PERM_WRITE); + List partitionWith8R8WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermRW, GRPC_TOPIC, GRPC_BROKER); + assertEquals(8, partitionWith8R8WPermRW.size()); + assertEquals(8, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count()); + assertEquals(0, partitionWith8R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + + // test queueData with 8 read queues, 8 write queues, and read only permission, expect 8 read only queues. + QueueData queueDataWith8R8WPermR = createQueueData(8, 8, PermName.PERM_READ); + List partitionWith8R8WPermR = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermR, GRPC_TOPIC, GRPC_BROKER); + assertEquals(8, partitionWith8R8WPermR.size()); + assertEquals(8, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.READ).count()); + assertEquals(0, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith8R8WPermR.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + + // test queueData with 8 read queues, 8 write queues, and write only permission, expect 8 write only queues. + QueueData queueDataWith8R8WPermW = createQueueData(8, 8, PermName.PERM_WRITE); + List partitionWith8R8WPermW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R8WPermW, GRPC_TOPIC, GRPC_BROKER); + assertEquals(8, partitionWith8R8WPermW.size()); + assertEquals(8, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + assertEquals(0, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith8R8WPermW.stream().filter(a -> a.getPermission() == Permission.READ).count()); + + // test queueData with 8 read queues, 0 write queues, and rw permission, expect 8 read only queues. + QueueData queueDataWith8R0WPermRW = createQueueData(8, 0, PermName.PERM_READ | PermName.PERM_WRITE); + List partitionWith8R0WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith8R0WPermRW, GRPC_TOPIC, GRPC_BROKER); + assertEquals(8, partitionWith8R0WPermRW.size()); + assertEquals(8, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count()); + assertEquals(0, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith8R0WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + + // test queueData with 4 read queues, 8 write queues, and rw permission, expect 4 rw queues and 4 write only queues. + QueueData queueDataWith4R8WPermRW = createQueueData(4, 8, PermName.PERM_READ | PermName.PERM_WRITE); + List partitionWith4R8WPermRW = this.routeActivity.genMessageQueueFromQueueData(queueDataWith4R8WPermRW, GRPC_TOPIC, GRPC_BROKER); + assertEquals(8, partitionWith4R8WPermRW.size()); + assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.WRITE).count()); + assertEquals(4, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ_WRITE).count()); + assertEquals(0, partitionWith4R8WPermRW.stream().filter(a -> a.getPermission() == Permission.READ).count()); + } + + private static QueueData createQueueData(int r, int w, int perm) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(BROKER_NAME); + queueData.setReadQueueNums(r); + queueData.setWriteQueueNums(w); + queueData.setPerm(perm); + return queueData; + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java new file mode 100644 index 0000000000..0a3992990a --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/transaction/EndTransactionActivityTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.grpc.v2.transaction; + +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.TransactionResolution; +import apache.rocketmq.v2.TransactionSource; +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +import org.apache.rocketmq.proxy.processor.TransactionStatus; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.when; + +@RunWith(Parameterized.class) +public class EndTransactionActivityTest extends BaseActivityTest { + + private EndTransactionActivity endTransactionActivity; + private TransactionResolution resolution; + private TransactionSource source; + private TransactionStatus transactionStatus; + private Boolean fromTransactionCheck; + + public EndTransactionActivityTest(TransactionResolution resolution, TransactionSource source, + TransactionStatus transactionStatus, Boolean fromTransactionCheck) { + this.resolution = resolution; + this.source = source; + this.transactionStatus = transactionStatus; + this.fromTransactionCheck = fromTransactionCheck; + } + + @Before + public void before() throws Throwable { + super.before(); + this.endTransactionActivity = new EndTransactionActivity(messagingProcessor, grpcClientSettingsManager, grpcChannelManager); + } + + @Test + public void testEndTransaction() throws Throwable { + ArgumentCaptor transactionStatusCaptor = ArgumentCaptor.forClass(TransactionStatus.class); + ArgumentCaptor fromTransactionCheckCaptor = ArgumentCaptor.forClass(Boolean.class); + when(this.messagingProcessor.endTransaction(any(), any(), anyString(), anyString(), + transactionStatusCaptor.capture(), + fromTransactionCheckCaptor.capture())).thenReturn(CompletableFuture.completedFuture(null)); + + EndTransactionResponse response = this.endTransactionActivity.endTransaction( + createContext(), + EndTransactionRequest.newBuilder() + .setResolution(resolution) + .setTopic(Resource.newBuilder().setName("topic").build()) + .setMessageId(MessageClientIDSetter.createUniqID()) + .setTransactionId(MessageClientIDSetter.createUniqID()) + .setSource(source) + .build() + ).get(); + + assertEquals(Code.OK, response.getStatus().getCode()); + assertEquals(transactionStatus, transactionStatusCaptor.getValue()); + assertEquals(fromTransactionCheck, fromTransactionCheckCaptor.getValue()); + } + + @Parameterized.Parameters + public static Collection parameters() { + Object[][] p = new Object[][] { + {TransactionResolution.COMMIT, TransactionSource.SOURCE_CLIENT, TransactionStatus.COMMIT, false}, + {TransactionResolution.ROLLBACK, TransactionSource.SOURCE_SERVER_CHECK, TransactionStatus.ROLLBACK, true}, + {TransactionResolution.TRANSACTION_RESOLUTION_UNSPECIFIED, TransactionSource.SOURCE_SERVER_CHECK, TransactionStatus.UNKNOWN, true}, + }; + return Arrays.asList(p); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java new file mode 100644 index 0000000000..a5c1d28365 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.nio.charset.StandardCharsets; +import java.util.Random; +import java.util.UUID; +import org.apache.rocketmq.broker.client.ConsumerManager; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.service.ServiceManager; +import org.apache.rocketmq.proxy.service.message.MessageService; +import org.apache.rocketmq.proxy.service.metadata.MetadataService; +import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.when; + +@Ignore +@RunWith(MockitoJUnitRunner.Silent.class) +public class BaseProcessorTest extends InitConfigAndLoggerTest { + protected static final Random RANDOM = new Random(); + + @Mock + protected MessagingProcessor messagingProcessor; + @Mock + protected ServiceManager serviceManager; + @Mock + protected MessageService messageService; + @Mock + protected TopicRouteService topicRouteService; + @Mock + protected ProducerManager producerManager; + @Mock + protected ConsumerManager consumerManager; + @Mock + protected TransactionService transactionService; + @Mock + protected ProxyRelayService proxyRelayService; + @Mock + protected MetadataService metadataService; + @Mock + protected ProducerProcessor producerProcessor; + @Mock + protected ConsumerProcessor consumerProcessor; + @Mock + protected TransactionProcessor transactionProcessor; + @Mock + protected ClientProcessor clientProcessor; + + public void before() throws Throwable { + super.before(); + when(serviceManager.getMessageService()).thenReturn(messageService); + when(serviceManager.getTopicRouteService()).thenReturn(topicRouteService); + when(serviceManager.getProducerManager()).thenReturn(producerManager); + when(serviceManager.getConsumerManager()).thenReturn(consumerManager); + when(serviceManager.getTransactionService()).thenReturn(transactionService); + when(serviceManager.getProxyRelayService()).thenReturn(proxyRelayService); + when(serviceManager.getMetadataService()).thenReturn(metadataService); + when(messagingProcessor.getMetadataService()).thenReturn(metadataService); + } + + protected static ProxyContext createContext() { + return ProxyContext.create(); + } + + protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { + MessageExt messageExt = new MessageExt(); + messageExt.setTopic(topic); + messageExt.setTags(tags); + messageExt.setReconsumeTimes(reconsumeTimes); + messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + messageExt.setMsgId(MessageClientIDSetter.createUniqID()); + messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, + ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, + RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); + return messageExt; + } + + protected static ReceiptHandle create(MessageExt messageExt) { + String ckInfo = messageExt.getProperty(MessageConst.PROPERTY_POP_CK); + if (ckInfo == null) { + return null; + } + return ReceiptHandle.decode(ckInfo + MessageConst.KEY_SEPARATOR + messageExt.getCommitLogOffset()); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java new file mode 100644 index 0000000000..dc7e969e78 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.constant.ConsumeInitMode; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.filter.ExpressionType; +import org.apache.rocketmq.common.filter.FilterAPI; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ConsumerProcessorTest extends BaseProcessorTest { + + private static final String CONSUMER_GROUP = "consumerGroup"; + private static final String TOPIC = "topic"; + + private ConsumerProcessor consumerProcessor; + + @Before + public void before() throws Throwable { + super.before(); + ReceiptHandleProcessor receiptHandleProcessor = new ReceiptHandleProcessor(messagingProcessor); + this.consumerProcessor = new ConsumerProcessor(messagingProcessor, serviceManager, Executors.newCachedThreadPool()); + } + + @Test + public void testPopMessage() throws Throwable { + final String tag = "tag"; + final long invisibleTime = Duration.ofSeconds(15).toMillis(); + ArgumentCaptor messageQueueArgumentCaptor = ArgumentCaptor.forClass(AddressableMessageQueue.class); + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(PopMessageRequestHeader.class); + + List messageExtList = new ArrayList<>(); + messageExtList.add(createMessageExt(TOPIC, "noMatch", 0, invisibleTime)); + messageExtList.add(createMessageExt(TOPIC, tag, 0, invisibleTime)); + messageExtList.add(createMessageExt(TOPIC, tag, 1, invisibleTime)); + PopResult innerPopResult = new PopResult(PopStatus.FOUND, messageExtList); + when(this.messageService.popMessage(any(), messageQueueArgumentCaptor.capture(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(innerPopResult)); + + when(this.topicRouteService.getCurrentMessageQueueView(anyString())) + .thenReturn(mock(MessageQueueView.class)); + + ArgumentCaptor ackMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + when(this.messagingProcessor.ackMessage(any(), any(), ackMessageIdArgumentCaptor.capture(), anyString(), anyString(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(mock(AckResult.class))); + + ArgumentCaptor toDLQMessageIdArgumentCaptor = ArgumentCaptor.forClass(String.class); + when(this.messagingProcessor.forwardMessageToDeadLetterQueue(any(), any(), toDLQMessageIdArgumentCaptor.capture(), anyString(), anyString(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(mock(RemotingCommand.class))); + + AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class); + PopResult popResult = this.consumerProcessor.popMessage( + createContext(), + (ctx, messageQueueView) -> messageQueue, + CONSUMER_GROUP, + TOPIC, + 60, + invisibleTime, + Duration.ofSeconds(3).toMillis(), + ConsumeInitMode.MAX, + FilterAPI.build(TOPIC, tag, ExpressionType.TAG), + false, + (ctx, consumerGroup, subscriptionData, messageExt) -> { + if (!messageExt.getTags().equals(tag)) { + return PopMessageResultFilter.FilterResult.NO_MATCH; + } + if (messageExt.getReconsumeTimes() > 0) { + return PopMessageResultFilter.FilterResult.TO_DLQ; + } + return PopMessageResultFilter.FilterResult.MATCH; + }, + Duration.ofSeconds(3).toMillis() + ).get(); + + assertSame(messageQueue, messageQueueArgumentCaptor.getValue()); + assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup()); + assertEquals(TOPIC, requestHeaderArgumentCaptor.getValue().getTopic()); + assertEquals(ProxyUtils.MAX_MSG_NUMS_FOR_POP_REQUEST, requestHeaderArgumentCaptor.getValue().getMaxMsgNums()); + assertEquals(tag, requestHeaderArgumentCaptor.getValue().getExp()); + assertEquals(ExpressionType.TAG, requestHeaderArgumentCaptor.getValue().getExpType()); + + assertEquals(PopStatus.FOUND, popResult.getPopStatus()); + assertEquals(1, popResult.getMsgFoundList().size()); + assertEquals(messageExtList.get(1), popResult.getMsgFoundList().get(0)); + + assertEquals(messageExtList.get(0).getMsgId(), ackMessageIdArgumentCaptor.getValue()); + assertEquals(messageExtList.get(2).getMsgId(), toDLQMessageIdArgumentCaptor.getValue()); + } + + @Test + public void testAckMessage() throws Throwable { + ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); + assertNotNull(handle); + + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(AckMessageRequestHeader.class); + AckResult innerAckResult = new AckResult(); + innerAckResult.setStatus(AckStatus.OK); + when(this.messageService.ackMessage(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(innerAckResult)); + + AckResult ackResult = this.consumerProcessor.ackMessage(createContext(), handle, MessageClientIDSetter.createUniqID(), + CONSUMER_GROUP, TOPIC, 3000).get(); + + assertEquals(AckStatus.OK, ackResult.getStatus()); + assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP), requestHeaderArgumentCaptor.getValue().getTopic()); + assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup()); + assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); + } + + @Test + public void testChangeInvisibleTime() throws Throwable { + ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); + assertNotNull(handle); + + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ChangeInvisibleTimeRequestHeader.class); + AckResult innerAckResult = new AckResult(); + innerAckResult.setStatus(AckStatus.OK); + when(this.messageService.changeInvisibleTime(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(innerAckResult)); + + AckResult ackResult = this.consumerProcessor.changeInvisibleTime(createContext(), handle, MessageClientIDSetter.createUniqID(), + CONSUMER_GROUP, TOPIC, 1000, 3000).get(); + + assertEquals(AckStatus.OK, ackResult.getStatus()); + assertEquals(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP), requestHeaderArgumentCaptor.getValue().getTopic()); + assertEquals(CONSUMER_GROUP, requestHeaderArgumentCaptor.getValue().getConsumerGroup()); + assertEquals(1000, requestHeaderArgumentCaptor.getValue().getInvisibleTime().longValue()); + assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java new file mode 100644 index 0000000000..690775e7c4 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ProducerProcessorTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.KeyBuilder; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageAccessor; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ProducerProcessorTest extends BaseProcessorTest { + + private static final String PRODUCER_GROUP = "producerGroup"; + private static final String CONSUMER_GROUP = "consumerGroup"; + private static final String TOPIC = "topic"; + + private ProducerProcessor producerProcessor; + + @Before + public void before() throws Throwable { + super.before(); + this.producerProcessor = new ProducerProcessor(this.messagingProcessor, this.serviceManager, Executors.newCachedThreadPool()); + } + + @Test + public void testSendMessage() throws Throwable { + when(metadataService.getTopicMessageType(eq(TOPIC))).thenReturn(TopicMessageType.NORMAL); + String txId = MessageClientIDSetter.createUniqID(); + String msgId = MessageClientIDSetter.createUniqID(); + long commitLogOffset = 1000L; + long queueOffset = 100L; + + SendResult sendResult = new SendResult(); + sendResult.setSendStatus(SendStatus.SEND_OK); + sendResult.setTransactionId(txId); + sendResult.setMsgId(msgId); + sendResult.setOffsetMsgId(createOffsetMsgId(commitLogOffset)); + sendResult.setQueueOffset(queueOffset); + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(SendMessageRequestHeader.class); + when(this.messageService.sendMessage(any(), any(), any(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult))); + + List messageList = new ArrayList<>(); + Message messageExt = createMessageExt(TOPIC, "tag", 0, 0); + messageList.add(messageExt); + AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class); + when(messageQueue.getBrokerName()).thenReturn("mockBroker"); + + ArgumentCaptor brokerNameCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor tranStateTableOffsetCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor commitLogOffsetCaptor = ArgumentCaptor.forClass(Long.class); + when(transactionService.addTransactionDataByBrokerName( + brokerNameCaptor.capture(), + anyString(), + tranStateTableOffsetCaptor.capture(), + commitLogOffsetCaptor.capture(), + anyString(), any())).thenReturn(mock(TransactionData.class)); + + List sendResultList = this.producerProcessor.sendMessage( + createContext(), + (ctx, messageQueueView) -> messageQueue, + PRODUCER_GROUP, + MessageSysFlag.TRANSACTION_PREPARED_TYPE, + messageList, + 3000 + ).get(); + + assertNotNull(sendResultList); + assertEquals("mockBroker", brokerNameCaptor.getValue()); + assertEquals(queueOffset, tranStateTableOffsetCaptor.getValue().longValue()); + assertEquals(commitLogOffset, commitLogOffsetCaptor.getValue().longValue()); + + SendMessageRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue(); + assertEquals(PRODUCER_GROUP, requestHeader.getProducerGroup()); + assertEquals(TOPIC, requestHeader.getTopic()); + } + + @Test + public void testSendRetryMessage() throws Throwable { + String txId = MessageClientIDSetter.createUniqID(); + String msgId = MessageClientIDSetter.createUniqID(); + long commitLogOffset = 1000L; + long queueOffset = 100L; + + SendResult sendResult = new SendResult(); + sendResult.setSendStatus(SendStatus.SEND_OK); + sendResult.setTransactionId(txId); + sendResult.setMsgId(msgId); + sendResult.setOffsetMsgId(createOffsetMsgId(commitLogOffset)); + sendResult.setQueueOffset(queueOffset); + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(SendMessageRequestHeader.class); + when(this.messageService.sendMessage(any(), any(), any(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(Lists.newArrayList(sendResult))); + + List messageExtList = new ArrayList<>(); + Message messageExt = createMessageExt(MixAll.getRetryTopic(CONSUMER_GROUP), "tag", 0, 0); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_RECONSUME_TIME, "1"); + MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_MAX_RECONSUME_TIMES, "16"); + messageExtList.add(messageExt); + AddressableMessageQueue messageQueue = mock(AddressableMessageQueue.class); + when(messageQueue.getBrokerName()).thenReturn("mockBroker"); + + ArgumentCaptor brokerNameCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor tranStateTableOffsetCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor commitLogOffsetCaptor = ArgumentCaptor.forClass(Long.class); + when(transactionService.addTransactionDataByBrokerName( + brokerNameCaptor.capture(), + anyString(), + tranStateTableOffsetCaptor.capture(), + commitLogOffsetCaptor.capture(), + anyString(), any())).thenReturn(mock(TransactionData.class)); + + List sendResultList = this.producerProcessor.sendMessage( + createContext(), + (ctx, messageQueueView) -> messageQueue, + PRODUCER_GROUP, + MessageSysFlag.TRANSACTION_PREPARED_TYPE, + messageExtList, + 3000 + ).get(); + + assertNotNull(sendResultList); + assertEquals("mockBroker", brokerNameCaptor.getValue()); + assertEquals(queueOffset, tranStateTableOffsetCaptor.getValue().longValue()); + assertEquals(commitLogOffset, commitLogOffsetCaptor.getValue().longValue()); + + SendMessageRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue(); + assertEquals(PRODUCER_GROUP, requestHeader.getProducerGroup()); + assertEquals(MixAll.getRetryTopic(CONSUMER_GROUP), requestHeader.getTopic()); + assertEquals(1, requestHeader.getReconsumeTimes().intValue()); + assertEquals(16, requestHeader.getMaxReconsumeTimes().intValue()); + } + + @Test + public void testForwardMessageToDeadLetterQueue() throws Throwable { + ArgumentCaptor requestHeaderArgumentCaptor = ArgumentCaptor.forClass(ConsumerSendMsgBackRequestHeader.class); + when(this.messageService.sendMessageBack(any(), any(), anyString(), requestHeaderArgumentCaptor.capture(), anyLong())) + .thenReturn(CompletableFuture.completedFuture(mock(RemotingCommand.class))); + + MessageExt messageExt = createMessageExt(KeyBuilder.buildPopRetryTopic(TOPIC, CONSUMER_GROUP), "", 16, 3000); + RemotingCommand remotingCommand = this.producerProcessor.forwardMessageToDeadLetterQueue( + createContext(), + create(messageExt), + messageExt.getMsgId(), + CONSUMER_GROUP, + TOPIC, + 3000 + ).get(); + + assertNotNull(remotingCommand); + ConsumerSendMsgBackRequestHeader requestHeader = requestHeaderArgumentCaptor.getValue(); + assertEquals(messageExt.getTopic(), requestHeader.getOriginTopic()); + assertEquals(messageExt.getMsgId(), requestHeader.getOriginMsgId()); + assertEquals(CONSUMER_GROUP, requestHeader.getGroup()); + } + + private static String createOffsetMsgId(long commitLogOffset) { + int msgIDLength = 4 + 4 + 8; + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(msgIDLength); + return MessageDecoder.createMessageId(byteBufferMsgId, + MessageExt.socketAddress2ByteBuffer(RemotingUtil.string2SocketAddress("127.0.0.1:10911")), + commitLogOffset); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessorTest.java new file mode 100644 index 0000000000..1f86332584 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ReceiptHandleProcessorTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.client.ClientChannelInfo; +import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.common.ContextVariable; +import org.apache.rocketmq.proxy.common.MessageReceiptHandle; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class ReceiptHandleProcessorTest extends BaseProcessorTest { + private ReceiptHandleProcessor receiptHandleProcessor; + + private static final ProxyContext PROXY_CONTEXT = ProxyContext.create(); + private static final String GROUP = "group"; + private static final String TOPIC = "topic"; + private static final String BROKER_NAME = "broker"; + private static final int QUEUE_ID = 1; + private static final String MESSAGE_ID = "messageId"; + private static final long OFFSET = 123L; + private static final long INVISIBLE_TIME = 100000L; + private static final int RECONSUME_TIMES = 1; + private static final String MSG_ID = MessageClientIDSetter.createUniqID(); + private MessageReceiptHandle messageReceiptHandle; + + private final String receiptHandle = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(0) + .invisibleTime(INVISIBLE_TIME) + .reviveQueueId(1) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(BROKER_NAME) + .queueId(QUEUE_ID) + .offset(OFFSET) + .commitLogOffset(0L) + .build().encode(); + + @Before + public void setup() { + PROXY_CONTEXT.withVal(ContextVariable.CLIENT_ID, "channel-id"); + receiptHandleProcessor = new ReceiptHandleProcessor(messagingProcessor); + Mockito.doNothing().when(messagingProcessor).registerConsumerListener(Mockito.any(ConsumerIdsChangeListener.class)); + messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, receiptHandle, MESSAGE_ID, OFFSET, + RECONSUME_TIMES, INVISIBLE_TIME); + } + + @Test + public void testAddReceiptHandle() { + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, GROUP, MSG_ID, receiptHandle, messageReceiptHandle); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(new SubscriptionGroupConfig()); + Mockito.when(messagingProcessor.findConsumerChannel(Mockito.any(), Mockito.eq(GROUP), Mockito.eq(channelId))).thenReturn(Mockito.mock(ClientChannelInfo.class)); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getRenewSliceTimeMillis())); + } + + @Test + public void testRenewReceiptHandle() { + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, GROUP, MSG_ID, receiptHandle, messageReceiptHandle); + SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig(); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(groupConfig); + Mockito.when(messagingProcessor.findConsumerChannel(Mockito.any(), Mockito.eq(GROUP), Mockito.eq(channelId))).thenReturn(Mockito.mock(ClientChannelInfo.class)); + long newInvisibleTime = 2000L; + ReceiptHandle newReceiptHandleClass = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(0) + .invisibleTime(newInvisibleTime) + .reviveQueueId(1) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(BROKER_NAME) + .queueId(QUEUE_ID) + .offset(OFFSET) + .commitLogOffset(0L) + .build(); + String newReceiptHandle = newReceiptHandleClass.encode(); + AckResult ackResult = new AckResult(); + ackResult.setStatus(AckStatus.OK); + ackResult.setExtraInfo(newReceiptHandle); + Mockito.when(messagingProcessor.changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getRenewSliceTimeMillis()))) + .thenReturn(CompletableFuture.completedFuture(ackResult)); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.argThat(r -> r.getInvisibleTime() == INVISIBLE_TIME), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getRenewSliceTimeMillis())); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.argThat(r -> r.getInvisibleTime() == newInvisibleTime), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getRenewSliceTimeMillis())); + } + + @Test + public void testRenewReceiptHandleWhenTimeout() { + long newInvisibleTime = 0L; + String newReceiptHandle = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(0) + .invisibleTime(newInvisibleTime) + .reviveQueueId(1) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(BROKER_NAME) + .queueId(QUEUE_ID) + .offset(OFFSET) + .commitLogOffset(0L) + .build().encode(); + messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, receiptHandle, MESSAGE_ID, OFFSET, + RECONSUME_TIMES, newInvisibleTime); + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, GROUP, MSG_ID, newReceiptHandle, messageReceiptHandle); + Mockito.when(messagingProcessor.findConsumerChannel(Mockito.any(), Mockito.eq(GROUP), Mockito.eq(channelId))).thenReturn(Mockito.mock(ClientChannelInfo.class)); + SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig(); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(groupConfig); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(groupConfig.getGroupRetryPolicy().getRetryPolicy().nextDelayDuration(RECONSUME_TIMES))); + } + + @Test + public void testRenewReceiptHandleWhenNotArrivingTime() { + String newReceiptHandle = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(System.currentTimeMillis()) + .invisibleTime(INVISIBLE_TIME) + .reviveQueueId(1) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(BROKER_NAME) + .queueId(QUEUE_ID) + .offset(OFFSET) + .commitLogOffset(0L) + .build().encode(); + messageReceiptHandle = new MessageReceiptHandle(GROUP, TOPIC, QUEUE_ID, newReceiptHandle, MESSAGE_ID, OFFSET, + RECONSUME_TIMES, INVISIBLE_TIME); + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, GROUP, MSG_ID, newReceiptHandle, messageReceiptHandle); + SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig(); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(groupConfig); + Mockito.when(messagingProcessor.findConsumerChannel(Mockito.any(), Mockito.eq(GROUP), Mockito.eq(channelId))).thenReturn(Mockito.mock(ClientChannelInfo.class)); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(0)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyLong()); + } + + @Test + public void testRemoveReceiptHandle() { + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, MSG_ID, receiptHandle, messageReceiptHandle); + receiptHandleProcessor.removeReceiptHandle(channelId, MSG_ID, receiptHandle); + SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig(); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(groupConfig); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(0)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyLong()); + } + + @Test + public void testClearGroup() { + String channelId = PROXY_CONTEXT.getVal(ContextVariable.CLIENT_ID); + receiptHandleProcessor.addReceiptHandle(channelId, MSG_ID, receiptHandle, messageReceiptHandle); + receiptHandleProcessor.clearGroup(channelId); + SubscriptionGroupConfig groupConfig = new SubscriptionGroupConfig(); + Mockito.when(metadataService.getSubscriptionGroupConfig(Mockito.eq(GROUP))).thenReturn(groupConfig); + receiptHandleProcessor.scheduleRenewTask(); + Mockito.verify(messagingProcessor, Mockito.timeout(1000).times(1)) + .changeInvisibleTime(Mockito.any(ProxyContext.class), Mockito.any(ReceiptHandle.class), Mockito.eq(MESSAGE_ID), + Mockito.eq(GROUP), Mockito.eq(TOPIC), Mockito.eq(ConfigurationManager.getProxyConfig().getInvisibleTimeMillisWhenClear())); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java new file mode 100644 index 0000000000..c21ad3dda3 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/TransactionProcessorTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.processor; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.service.transaction.EndTransactionRequestData; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +public class TransactionProcessorTest extends BaseProcessorTest { + + private static final String PRODUCER_GROUP = "producerGroup"; + private TransactionProcessor transactionProcessor; + + @Before + public void before() throws Throwable { + super.before(); + this.transactionProcessor = new TransactionProcessor(this.messagingProcessor, this.serviceManager); + } + + @Test + public void testEndTransaction() throws Throwable { + testEndTransaction(MessageSysFlag.TRANSACTION_COMMIT_TYPE, TransactionStatus.COMMIT); + testEndTransaction(MessageSysFlag.TRANSACTION_NOT_TYPE, TransactionStatus.UNKNOWN); + testEndTransaction(MessageSysFlag.TRANSACTION_ROLLBACK_TYPE, TransactionStatus.ROLLBACK); + } + + protected void testEndTransaction(int sysFlag, TransactionStatus transactionStatus) throws Throwable { + when(this.messageService.endTransactionOneway(any(), any(), any(), anyLong())).thenReturn(CompletableFuture.completedFuture(null)); + ArgumentCaptor commitOrRollbackCaptor = ArgumentCaptor.forClass(Integer.class); + when(transactionService.genEndTransactionRequestHeader(anyString(), commitOrRollbackCaptor.capture(), anyBoolean(), anyString(), anyString())) + .thenReturn(new EndTransactionRequestData("brokerName", new EndTransactionRequestHeader())); + + this.transactionProcessor.endTransaction( + createContext(), + "transactionId", + "msgId", + PRODUCER_GROUP, + transactionStatus, + true, + 3000 + ); + + assertEquals(sysFlag, commitOrRollbackCaptor.getValue().intValue()); + + reset(this.messageService); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java new file mode 100644 index 0000000000..f6ca31ff84 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service; + +import java.util.HashMap; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.common.protocol.route.QueueData; +import org.apache.rocketmq.common.protocol.route.TopicRouteData; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIExt; +import org.apache.rocketmq.proxy.service.mqclient.MQClientAPIFactory; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.apache.rocketmq.proxy.service.route.TopicRouteService; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@Ignore +@RunWith(MockitoJUnitRunner.Silent.class) +public class BaseServiceTest extends InitConfigAndLoggerTest { + + protected TopicRouteService topicRouteService; + protected MQClientAPIFactory mqClientAPIFactory; + protected MQClientAPIExt mqClientAPIExt; + + protected static final String ERR_TOPIC = "errTopic"; + protected static final String TOPIC = "topic"; + protected static final String GROUP = "group"; + protected static final String BROKER_NAME = "broker"; + protected static final String CLUSTER_NAME = "cluster"; + protected static final String BROKER_ADDR = "127.0.0.1:10911"; + + protected final TopicRouteData topicRouteData = new TopicRouteData(); + protected final QueueData queueData = new QueueData(); + protected final BrokerData brokerData = new BrokerData(); + + @Before + public void before() throws Throwable { + super.before(); + + topicRouteService = mock(TopicRouteService.class); + mqClientAPIFactory = mock(MQClientAPIFactory.class); + mqClientAPIExt = mock(MQClientAPIExt.class); + when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt); + + queueData.setBrokerName(BROKER_NAME); + topicRouteData.setQueueDatas(Lists.newArrayList(queueData)); + brokerData.setCluster(CLUSTER_NAME); + brokerData.setBrokerName(BROKER_NAME); + HashMap brokerAddrs = new HashMap<>(); + brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); + brokerData.setBrokerAddrs(brokerAddrs); + topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); + + when(this.topicRouteService.getAllMessageQueueView(eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); + when(this.topicRouteService.getAllMessageQueueView(eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); + when(this.topicRouteService.getAllMessageQueueView(eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData)); + } +} diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java new file mode 100644 index 0000000000..e3f6edb997 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/message/LocalMessageServiceTest.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.message; + +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.processor.AckMessageProcessor; +import org.apache.rocketmq.broker.processor.ChangeInvisibleTimeProcessor; +import org.apache.rocketmq.broker.processor.EndTransactionProcessor; +import org.apache.rocketmq.broker.processor.PopMessageProcessor; +import org.apache.rocketmq.broker.processor.SendMessageProcessor; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.AckStatus; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.consumer.ReceiptHandle; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageBatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.message.MessageQueue; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeResponseHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import org.apache.rocketmq.common.protocol.header.ExtraInfoUtil; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageResponseHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.proxy.common.ContextVariable; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.common.ProxyException; +import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.apache.rocketmq.proxy.service.channel.ChannelManager; +import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext; +import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; +import org.apache.rocketmq.remoting.exception.RemotingCommandException; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +@RunWith(MockitoJUnitRunner.class) +public class LocalMessageServiceTest extends InitConfigAndLoggerTest { + private LocalMessageService localMessageService; + @Mock + private SendMessageProcessor sendMessageProcessorMock; + @Mock + private EndTransactionProcessor endTransactionProcessorMock; + @Mock + private PopMessageProcessor popMessageProcessorMock; + @Mock + private ChangeInvisibleTimeProcessor changeInvisibleTimeProcessorMock; + @Mock + private AckMessageProcessor ackMessageProcessorMock; + @Mock + private BrokerController brokerControllerMock; + + private ProxyContext proxyContext; + + private ChannelManager channelManager; + + private String topic = "topic"; + + private String brokerName = "brokerName"; + + private int queueId = 0; + + private long queueOffset = 0L; + + private String transactionId = "transactionId"; + + private String offsetMessageId = "offsetMessageId"; + + @Before + public void setUp() throws Throwable { + super.before(); + ConfigurationManager.getProxyConfig().setNameSrvAddr("1.1.1.1"); + channelManager = new ChannelManager(); + Mockito.when(brokerControllerMock.getSendMessageProcessor()).thenReturn(sendMessageProcessorMock); + Mockito.when(brokerControllerMock.getPopMessageProcessor()).thenReturn(popMessageProcessorMock); + Mockito.when(brokerControllerMock.getChangeInvisibleTimeProcessor()).thenReturn(changeInvisibleTimeProcessorMock); + Mockito.when(brokerControllerMock.getAckMessageProcessor()).thenReturn(ackMessageProcessorMock); + Mockito.when(brokerControllerMock.getEndTransactionProcessor()).thenReturn(endTransactionProcessorMock); + Mockito.when(brokerControllerMock.getBrokerConfig()).thenReturn(new BrokerConfig()); + localMessageService = new LocalMessageService(brokerControllerMock, channelManager, null); + proxyContext = ProxyContext.create().withVal(ContextVariable.REMOTE_ADDRESS, "0.0.0.1") + .withVal(ContextVariable.LOCAL_ADDRESS, "0.0.0.2"); + } + + @Test + public void testSendMessageWriteAndFlush() throws Exception { + Message message = new Message(topic, "body".getBytes(StandardCharsets.UTF_8)); + MessageClientIDSetter.setUniqID(message); + List messagesList = Collections.singletonList(message); + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.SEND_MESSAGE; + boolean second = Arrays.equals(argument.getBody(), message.getBody()); + return first & second; + }))).thenAnswer(invocation -> { + SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0); + RemotingCommand request = invocation.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + response.setBody(message.getBody()); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setQueueId(queueId); + sendMessageResponseHeader.setQueueOffset(queueOffset); + sendMessageResponseHeader.setMsgId(offsetMessageId); + sendMessageResponseHeader.setTransactionId(transactionId); + simpleChannelHandlerContext.writeAndFlush(response); + return null; + }); + + CompletableFuture> future = localMessageService.sendMessage(proxyContext, null, messagesList, requestHeader, 1000L); + SendResult sendResult = future.get().get(0); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getMsgId()).isEqualTo(MessageClientIDSetter.getUniqID(message)); + assertThat(sendResult.getMessageQueue()) + .isEqualTo(new MessageQueue(topic, brokerControllerMock.getBrokerConfig().getBrokerName(), queueId)); + assertThat(sendResult.getQueueOffset()).isEqualTo(queueOffset); + assertThat(sendResult.getTransactionId()).isEqualTo(transactionId); + assertThat(sendResult.getOffsetMsgId()).isEqualTo(offsetMessageId); + } + + @Test + public void testSendBatchMessageWriteAndFlush() throws Exception { + Message message1 = new Message(topic, "body1".getBytes(StandardCharsets.UTF_8)); + Message message2 = new Message(topic, "body2".getBytes(StandardCharsets.UTF_8)); + MessageClientIDSetter.setUniqID(message1); + MessageClientIDSetter.setUniqID(message2); + List messagesList = Arrays.asList(message1, message2); + MessageBatch msgBatch = MessageBatch.generateFromList(messagesList); + MessageClientIDSetter.setUniqID(msgBatch); + byte[] body = msgBatch.encode(); + msgBatch.setBody(body); + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queueId); + Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.SEND_MESSAGE; + boolean second = Arrays.equals(argument.getBody(), body); + return first & second; + }))).thenAnswer(invocation -> { + SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0); + RemotingCommand request = invocation.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + response.setBody(body); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setQueueId(queueId); + sendMessageResponseHeader.setQueueOffset(queueOffset); + sendMessageResponseHeader.setMsgId(offsetMessageId); + sendMessageResponseHeader.setTransactionId(transactionId); + simpleChannelHandlerContext.writeAndFlush(response); + return null; + }); + + CompletableFuture> future = localMessageService.sendMessage(proxyContext, null, messagesList, requestHeader, 1000L); + SendResult sendResult = future.get().get(0); + assertThat(sendResult.getSendStatus()).isEqualTo(SendStatus.SEND_OK); + assertThat(sendResult.getMessageQueue()) + .isEqualTo(new MessageQueue(topic, brokerControllerMock.getBrokerConfig().getBrokerName(), queueId)); + assertThat(sendResult.getQueueOffset()).isEqualTo(queueOffset); + assertThat(sendResult.getTransactionId()).isEqualTo(transactionId); + assertThat(sendResult.getOffsetMsgId()).isEqualTo(offsetMessageId); + } + + @Test + public void testSendMessageError() throws Exception { + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + Message message = new Message("topic", "body".getBytes(StandardCharsets.UTF_8)); + MessageClientIDSetter.setUniqID(message); + List messagesList = Collections.singletonList(message); + SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader(); + sendMessageRequestHeader.setTopic(topic); + sendMessageRequestHeader.setQueueId(queueId); + + Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.any(RemotingCommand.class))) + .thenReturn(response); + + CompletableFuture> future = localMessageService.sendMessage(proxyContext, null, messagesList, sendMessageRequestHeader, 1000L); + ExecutionException exception = catchThrowableOfType(future::get, ExecutionException.class); + assertThat(exception.getCause()).isInstanceOf(ProxyException.class); + assertThat(((ProxyException) exception.getCause()).getCode()).isEqualTo(ProxyExceptionCode.INTERNAL_SERVER_ERROR); + } + + @Test + public void testSendMessageWithException() throws Exception { + Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.any(RemotingCommand.class))) + .thenThrow(new RemotingCommandException("test")); + Message message = new Message("topic", "body".getBytes(StandardCharsets.UTF_8)); + MessageClientIDSetter.setUniqID(message); + List messagesList = Collections.singletonList(message); + SendMessageRequestHeader sendMessageRequestHeader = new SendMessageRequestHeader(); + CompletableFuture> future = localMessageService.sendMessage(proxyContext, null, messagesList, sendMessageRequestHeader, 1000L); + ExecutionException exception = catchThrowableOfType(future::get, ExecutionException.class); + assertThat(exception.getCause()).isInstanceOf(RemotingCommandException.class); + } + + @Test + public void testSendMessageBack() throws Exception { + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, ""); + Mockito.when(sendMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.CONSUMER_SEND_MSG_BACK; + boolean second = argument.readCustomHeader() instanceof ConsumerSendMsgBackRequestHeader; + return first && second; + }))).thenReturn(remotingCommand); + ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader(); + CompletableFuture future = localMessageService.sendMessageBack(proxyContext, null, null, requestHeader, 1000L); + RemotingCommand response = future.get(); + assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); + } + + @Test + public void testEndTransaction() throws Exception { + EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); + localMessageService.endTransactionOneway(proxyContext, null, requestHeader, 1000L); + Mockito.verify(endTransactionProcessorMock, Mockito.times(1)).processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.END_TRANSACTION; + boolean second = argument.readCustomHeader() instanceof EndTransactionRequestHeader; + return first && second; + })); + } + + @Test + public void testPopMessageWriteAndFlush() throws Exception { + int reviveQueueId = 1; + long popTime = System.currentTimeMillis(); + long invisibleTime = 3000L; + long startOffset = 100L; + long restNum = 0L; + StringBuilder startOffsetStringBuilder = new StringBuilder(); + StringBuilder messageOffsetStringBuilder = new StringBuilder(); + List messageExtList = new ArrayList<>(); + List messageOffsetList = new ArrayList<>(); + MessageExt message1 = buildMessageExt(topic, 0, startOffset); + messageExtList.add(message1); + messageOffsetList.add(startOffset); + byte[] body1 = MessageDecoder.encode(message1, false); + MessageExt message2 = buildMessageExt(topic, 0, startOffset + 1); + messageExtList.add(message2); + messageOffsetList.add(startOffset + 1); + ExtraInfoUtil.buildStartOffsetInfo(startOffsetStringBuilder, false, queueId, startOffset); + ExtraInfoUtil.buildMsgOffsetInfo(messageOffsetStringBuilder, false, queueId, messageOffsetList); + byte[] body2 = MessageDecoder.encode(message2, false); + ByteBuffer byteBuffer1 = ByteBuffer.wrap(body1); + ByteBuffer byteBuffer2 = ByteBuffer.wrap(body2); + ByteBuffer b3 = ByteBuffer.allocate(byteBuffer1.limit() + byteBuffer2.limit()); + b3.put(byteBuffer1); + b3.put(byteBuffer2); + PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); + requestHeader.setInvisibleTime(invisibleTime); + Mockito.when(popMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.POP_MESSAGE; + boolean second = argument.readCustomHeader() instanceof PopMessageRequestHeader; + return first && second; + }))).thenAnswer(invocation -> { + SimpleChannelHandlerContext simpleChannelHandlerContext = invocation.getArgument(0); + RemotingCommand request = invocation.getArgument(1); + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + response.setOpaque(request.getOpaque()); + response.setCode(ResponseCode.SUCCESS); + response.setBody(b3.array()); + PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + responseHeader.setStartOffsetInfo(startOffsetStringBuilder.toString()); + responseHeader.setMsgOffsetInfo(messageOffsetStringBuilder.toString()); + responseHeader.setInvisibleTime(requestHeader.getInvisibleTime()); + responseHeader.setPopTime(popTime); + responseHeader.setRestNum(restNum); + responseHeader.setReviveQid(reviveQueueId); + simpleChannelHandlerContext.writeAndFlush(response); + return null; + }); + MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId); + CompletableFuture future = localMessageService.popMessage(proxyContext, new AddressableMessageQueue(messageQueue, ""), requestHeader, 1000L); + PopResult popResult = future.get(); + assertThat(popResult.getPopTime()).isEqualTo(popTime); + assertThat(popResult.getInvisibleTime()).isEqualTo(invisibleTime); + assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.FOUND); + assertThat(popResult.getRestNum()).isEqualTo(restNum); + assertThat(popResult.getMsgFoundList().size()).isEqualTo(messageExtList.size()); + for (int i = 0; i < popResult.getMsgFoundList().size(); i++) { + assertMessageExt(popResult.getMsgFoundList().get(i), messageExtList.get(i)); + } + } + + @Test + public void testPopMessagePollingTimeout() throws Exception { + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.POLLING_TIMEOUT, ""); + Mockito.when(popMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.POP_MESSAGE; + boolean second = argument.readCustomHeader() instanceof PopMessageRequestHeader; + return first && second; + }))).thenReturn(remotingCommand); + PopMessageRequestHeader requestHeader = new PopMessageRequestHeader(); + CompletableFuture future = localMessageService.popMessage(proxyContext, null, requestHeader, 1000L); + PopResult popResult = future.get(); + assertThat(popResult.getPopStatus()).isEqualTo(PopStatus.POLLING_NOT_FOUND); + } + + @Test + public void testChangeInvisibleTime() throws Exception { + String messageId = "messageId"; + long popTime = System.currentTimeMillis(); + long invisibleTime = 3000L; + int reviveQueueId = 1; + ReceiptHandle handle = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(popTime) + .invisibleTime(invisibleTime) + .reviveQueueId(reviveQueueId) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(brokerName) + .queueId(queueId) + .offset(queueOffset) + .build(); + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ChangeInvisibleTimeResponseHeader.class); + remotingCommand.setCode(ResponseCode.SUCCESS); + remotingCommand.setRemark(""); + long newPopTime = System.currentTimeMillis(); + long newInvisibleTime = 5000L; + int newReviveQueueId = 2; + ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) remotingCommand.readCustomHeader(); + responseHeader.setReviveQid(newReviveQueueId); + responseHeader.setInvisibleTime(newInvisibleTime); + responseHeader.setPopTime(newPopTime); + Mockito.when(changeInvisibleTimeProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.CHANGE_MESSAGE_INVISIBLETIME; + boolean second = argument.readCustomHeader() instanceof ChangeInvisibleTimeRequestHeader; + return first && second; + }))).thenReturn(remotingCommand); + ChangeInvisibleTimeRequestHeader requestHeader = new ChangeInvisibleTimeRequestHeader(); + CompletableFuture future = localMessageService.changeInvisibleTime(proxyContext, handle, messageId, + requestHeader, 1000L); + AckResult ackResult = future.get(); + assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK); + assertThat(ackResult.getPopTime()).isEqualTo(newPopTime); + assertThat(ackResult.getExtraInfo()).isEqualTo(ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(newPopTime) + .invisibleTime(newInvisibleTime) + .reviveQueueId(newReviveQueueId) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(brokerName) + .queueId(queueId) + .offset(queueOffset) + .build() + .encode()); + } + + @Test + public void testAckMessage() throws Exception { + String messageId = "messageId"; + long popTime = System.currentTimeMillis(); + long invisibleTime = 3000L; + int reviveQueueId = 1; + ReceiptHandle handle = ReceiptHandle.builder() + .startOffset(0L) + .retrieveTime(popTime) + .invisibleTime(invisibleTime) + .reviveQueueId(reviveQueueId) + .topicType(ReceiptHandle.NORMAL_TOPIC) + .brokerName(brokerName) + .queueId(queueId) + .offset(queueOffset) + .build(); + RemotingCommand remotingCommand = RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, null); + Mockito.when(ackMessageProcessorMock.processRequest(Mockito.any(SimpleChannelHandlerContext.class), Mockito.argThat(argument -> { + boolean first = argument.getCode() == RequestCode.ACK_MESSAGE; + boolean second = argument.readCustomHeader() instanceof AckMessageRequestHeader; + return first && second; + }))).thenReturn(remotingCommand); + AckMessageRequestHeader requestHeader = new AckMessageRequestHeader(); + CompletableFuture future = localMessageService.ackMessage(proxyContext, handle, messageId, + requestHeader, 1000L); + AckResult ackResult = future.get(); + assertThat(ackResult.getStatus()).isEqualTo(AckStatus.OK); + } + + private MessageExt buildMessageExt(String topic, int queueId, long queueOffset) { + MessageExt message1 = new MessageExt(); + message1.setTopic(topic); + message1.setBody("body".getBytes(StandardCharsets.UTF_8)); + message1.setFlag(0); + message1.setQueueId(queueId); + message1.setQueueOffset(queueOffset); + message1.setCommitLogOffset(1000L); + message1.setSysFlag(0); + message1.setBornTimestamp(0L); + InetSocketAddress inetSocketAddress = new InetSocketAddress("127.0.0.1", 80); + message1.setBornHost(inetSocketAddress); + message1.setStoreHost(inetSocketAddress); + message1.setReconsumeTimes(0); + message1.setPreparedTransactionOffset(0L); + message1.putUserProperty("K", "V"); + return message1; + } + + private void assertMessageExt(MessageExt messageExt1, MessageExt messageExt2) { + assertThat(messageExt1.getBody()).isEqualTo(messageExt2.getBody()); + assertThat(messageExt1.getTopic()).isEqualTo(messageExt2.getTopic()); + assertThat(messageExt1.getQueueId()).isEqualTo(messageExt2.getQueueId()); + assertThat(messageExt1.getQueueOffset()).isEqualTo(messageExt2.getQueueOffset()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java new file mode 100644 index 0000000000..2c0d3f8909 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataServiceTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.metadata; + +import java.util.HashMap; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.statictopic.TopicConfigAndQueueMapping; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.service.BaseServiceTest; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +public class ClusterMetadataServiceTest extends BaseServiceTest { + + private ClusterMetadataService clusterMetadataService; + + @Before + public void before() throws Throwable { + super.before(); + ConfigurationManager.getProxyConfig().setRocketMQClusterName(CLUSTER_NAME); + + TopicConfigAndQueueMapping topicConfigAndQueueMapping = new TopicConfigAndQueueMapping(); + topicConfigAndQueueMapping.setAttributes(new HashMap<>()); + topicConfigAndQueueMapping.setTopicMessageType(TopicMessageType.NORMAL); + when(this.mqClientAPIExt.getTopicConfig(anyString(), eq(TOPIC), anyLong())).thenReturn(topicConfigAndQueueMapping); + + when(this.mqClientAPIExt.getSubscriptionGroupConfig(anyString(), eq(GROUP), anyLong())).thenReturn(new SubscriptionGroupConfig()); + + this.clusterMetadataService = new ClusterMetadataService(this.topicRouteService, this.mqClientAPIFactory); + } + + @Test + public void testGetTopicMessageType() { + assertEquals(TopicMessageType.UNSPECIFIED, this.clusterMetadataService.getTopicMessageType(ERR_TOPIC)); + assertEquals(1, this.clusterMetadataService.topicConfigCache.asMap().size()); + assertEquals(TopicMessageType.UNSPECIFIED, this.clusterMetadataService.getTopicMessageType(ERR_TOPIC)); + + assertEquals(TopicMessageType.NORMAL, this.clusterMetadataService.getTopicMessageType(TOPIC)); + assertEquals(2, this.clusterMetadataService.topicConfigCache.asMap().size()); + } + + @Test + public void testGetSubscriptionGroupConfig() { + assertNotNull(this.clusterMetadataService.getSubscriptionGroupConfig(GROUP)); + assertEquals(1, this.clusterMetadataService.subscriptionGroupConfigCache.asMap().size()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java new file mode 100644 index 0000000000..fad425420d --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java @@ -0,0 +1,353 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.mqclient; + +import java.lang.reflect.Field; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import org.apache.rocketmq.client.ClientConfig; +import org.apache.rocketmq.client.consumer.AckCallback; +import org.apache.rocketmq.client.consumer.AckResult; +import org.apache.rocketmq.client.consumer.PopCallback; +import org.apache.rocketmq.client.consumer.PopResult; +import org.apache.rocketmq.client.consumer.PopStatus; +import org.apache.rocketmq.client.consumer.PullCallback; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.client.impl.CommunicationMode; +import org.apache.rocketmq.client.impl.MQClientAPIImpl; +import org.apache.rocketmq.client.impl.consumer.PullResultExt; +import org.apache.rocketmq.client.producer.SendResult; +import org.apache.rocketmq.client.producer.SendStatus; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.header.AckMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.ChangeInvisibleTimeRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody; +import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.PopMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SearchOffsetRequestHeader; +import org.apache.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; +import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.remoting.InvokeCallback; +import org.apache.rocketmq.remoting.RemotingClient; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.netty.NettyClientConfig; +import org.apache.rocketmq.remoting.netty.ResponseFuture; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; + +@RunWith(MockitoJUnitRunner.class) +public class MQClientAPIExtTest { + + private static final String BROKER_ADDR = "127.0.0.1:10911"; + private static final String BROKER_NAME = "brokerName"; + private static final long TIMEOUT = 3000; + private static final String CONSUMER_GROUP = "group"; + private static final String TOPIC = "topic"; + + @Spy + private final MQClientAPIExt mqClientAPI = new MQClientAPIExt(new ClientConfig(), new NettyClientConfig(), new DoNothingClientRemotingProcessor(null), null); + @Mock + private RemotingClient remotingClient; + + @Before + public void init() throws Exception { + Field field = MQClientAPIImpl.class.getDeclaredField("remotingClient"); + field.setAccessible(true); + field.set(mqClientAPI, remotingClient); + } + + @Test + public void testSendHeartbeatAsync() throws Exception { + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); + } + + @Test + public void testSendMessageAsync() throws Exception { + AtomicReference msgIdRef = new AtomicReference<>(); + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(msgIdRef.get()); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + MessageExt messageExt = createMessage(); + msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); + + SendResult sendResult = mqClientAPI.sendMessageAsync(BROKER_ADDR, BROKER_NAME, messageExt, new SendMessageRequestHeader(), TIMEOUT) + .get(); + assertNotNull(sendResult); + assertEquals(msgIdRef.get(), sendResult.getMsgId()); + assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus()); + } + + @Test + public void testSendMessageListAsync() throws Exception { + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); + sendMessageResponseHeader.setMsgId(""); + sendMessageResponseHeader.setQueueId(0); + sendMessageResponseHeader.setQueueOffset(1L); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + List messageExtList = new ArrayList<>(); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 3; i++) { + MessageExt messageExt = createMessage(); + sb.append(sb.length() == 0 ? "" : ",").append(MessageClientIDSetter.getUniqID(messageExt)); + messageExtList.add(messageExt); + } + + SendResult sendResult = mqClientAPI.sendMessageAsync(BROKER_ADDR, BROKER_NAME, messageExtList, new SendMessageRequestHeader(), TIMEOUT) + .get(); + assertNotNull(sendResult); + assertEquals(sb.toString(), sendResult.getMsgId()); + assertEquals(SendStatus.SEND_OK, sendResult.getSendStatus()); + } + + @Test + public void testSendMessageBackAsync() throws Exception { + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) + .get(); + assertNotNull(remotingCommand); + assertEquals(ResponseCode.SUCCESS, remotingCommand.getCode()); + } + + @Test + public void testPopMessageAsync() throws Exception { + PopResult popResult = new PopResult(PopStatus.POLLING_NOT_FOUND, null); + doAnswer((Answer) mock -> { + PopCallback popCallback = mock.getArgument(4); + popCallback.onSuccess(popResult); + return null; + }).when(mqClientAPI).popMessageAsync(anyString(), anyString(), any(), anyLong(), any()); + + assertSame(popResult, mqClientAPI.popMessageAsync(BROKER_ADDR, BROKER_NAME, new PopMessageRequestHeader(), TIMEOUT).get()); + } + + @Test + public void testAckMessageAsync() throws Exception { + AckResult ackResult = new AckResult(); + doAnswer((Answer) mock -> { + AckCallback ackCallback = mock.getArgument(2); + ackCallback.onSuccess(ackResult); + return null; + }).when(mqClientAPI).ackMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); + + assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); + } + + @Test + public void testChangeInvisibleTimeAsync() throws Exception { + AckResult ackResult = new AckResult(); + doAnswer((Answer) mock -> { + AckCallback ackCallback = mock.getArgument(4); + ackCallback.onSuccess(ackResult); + return null; + }).when(mqClientAPI).changeInvisibleTimeAsync(anyString(), anyString(), any(), anyLong(), any(AckCallback.class)); + + assertSame(ackResult, mqClientAPI.changeInvisibleTimeAsync(BROKER_ADDR, BROKER_NAME, new ChangeInvisibleTimeRequestHeader(), TIMEOUT).get()); + } + + @Test + public void testPullMessageAsync() throws Exception { + MessageExt msg1 = createMessage(); + byte[] msg1Byte = MessageDecoder.encode(msg1, false); + MessageExt msg2 = createMessage(); + byte[] msg2Byte = MessageDecoder.encode(msg2, false); + + ByteBuffer byteBuffer = ByteBuffer.allocate(msg1Byte.length + msg2Byte.length); + byteBuffer.put(msg1Byte); + byteBuffer.put(msg2Byte); + + PullResultExt pullResultExt = new PullResultExt(PullStatus.FOUND, 0, 0, 1, null, 0, + byteBuffer.array()); + doAnswer((Answer) mock -> { + PullCallback pullCallback = mock.getArgument(4); + pullCallback.onSuccess(pullResultExt); + return null; + }).when(mqClientAPI).pullMessage(anyString(), any(), anyLong(), any(CommunicationMode.class), any(PullCallback.class)); + + PullResult pullResult = mqClientAPI.pullMessageAsync(BROKER_ADDR, new PullMessageRequestHeader(), TIMEOUT).get(); + assertNotNull(pullResult); + assertEquals(2, pullResult.getMsgFoundList().size()); + + Set msgIdSet = pullResult.getMsgFoundList().stream().map(MessageClientIDSetter::getUniqID).collect(Collectors.toSet()); + assertTrue(msgIdSet.contains(MessageClientIDSetter.getUniqID(msg1))); + assertTrue(msgIdSet.contains(MessageClientIDSetter.getUniqID(msg2))); + } + + @Test + public void testGetConsumerListByGroupAsync() throws Exception { + List clientIds = Lists.newArrayList("clientIds"); + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody(); + body.setConsumerIdList(clientIds); + response.setBody(body.encode()); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + List res = mqClientAPI.getConsumerListByGroupAsync(BROKER_ADDR, new GetConsumerListByGroupRequestHeader(), TIMEOUT).get(); + assertEquals(clientIds, res); + } + + @Test + public void testGetEmptyConsumerListByGroupAsync() throws Exception { + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(GetConsumerListByGroupRequestHeader.class); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + List res = mqClientAPI.getConsumerListByGroupAsync(BROKER_ADDR, new GetConsumerListByGroupRequestHeader(), TIMEOUT).get(); + assertTrue(res.isEmpty()); + } + + @Test + public void testGetMaxOffsetAsync() throws Exception { + long offset = ThreadLocalRandom.current().nextLong(); + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); + GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); + requestHeader.setTopic(TOPIC); + requestHeader.setQueueId(0); + assertEquals(offset, mqClientAPI.getMaxOffset(BROKER_ADDR, requestHeader, TIMEOUT).get().longValue()); + } + + @Test + public void testSearchOffsetAsync() throws Exception { + long offset = ThreadLocalRandom.current().nextLong(); + doAnswer((Answer) mock -> { + InvokeCallback invokeCallback = mock.getArgument(3); + ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); + RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.makeCustomHeaderToNet(); + responseFuture.putResponse(response); + invokeCallback.operationComplete(responseFuture); + return null; + }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); + + SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); + requestHeader.setTopic(TOPIC); + requestHeader.setQueueId(0); + requestHeader.setTimestamp(System.currentTimeMillis()); + assertEquals(offset, mqClientAPI.searchOffset(BROKER_ADDR, requestHeader, TIMEOUT).get().longValue()); + } + + protected MessageExt createMessage() { + MessageExt messageExt = new MessageExt(); + messageExt.setTopic("topic"); + messageExt.setBornHost(RemotingUtil.string2SocketAddress("127.0.0.2:8888")); + messageExt.setStoreHost(RemotingUtil.string2SocketAddress("127.0.0.1:10911")); + messageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + MessageClientIDSetter.setUniqID(messageExt); + return messageExt; + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayServiceTest.java new file mode 100644 index 0000000000..5f18188e5d --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/LocalProxyRelayServiceTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.common.protocol.body.CMResult; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.service.channel.SimpleChannelHandlerContext; +import org.apache.rocketmq.proxy.service.transaction.TransactionService; +import org.apache.rocketmq.remoting.netty.NettyRemotingServer; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(MockitoJUnitRunner.class) +public class LocalProxyRelayServiceTest { + private LocalProxyRelayService localProxyRelayService; + @Mock + private BrokerController brokerControllerMock; + @Mock + private TransactionService transactionService; + @Mock + private NettyRemotingServer nettyRemotingServerMock; + + @Before + public void setUp() { + localProxyRelayService = new LocalProxyRelayService(brokerControllerMock, transactionService); + Mockito.when(brokerControllerMock.getRemotingServer()).thenReturn(nettyRemotingServerMock); + } + + @Test + public void testProcessGetConsumerRunningInfo() { + ConsumerRunningInfo runningInfo = new ConsumerRunningInfo(); + runningInfo.setJstack("jstack"); + String remark = "ok"; + int opaque = 123; + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, null); + remotingCommand.setOpaque(opaque); + GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader(); + requestHeader.setJstackEnable(true); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(RemotingCommand.class); + CompletableFuture> future = + localProxyRelayService.processGetConsumerRunningInfo(ProxyContext.create(), remotingCommand, requestHeader); + future.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, remark, runningInfo)); + Mockito.verify(nettyRemotingServerMock, Mockito.times(1)) + .processResponseCommand(Mockito.any(SimpleChannelHandlerContext.class), argumentCaptor.capture()); + RemotingCommand remotingCommand1 = argumentCaptor.getValue(); + assertThat(remotingCommand1.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(remotingCommand1.getRemark()).isEqualTo(remark); + assertThat(remotingCommand1.getBody()).isEqualTo(runningInfo.encode()); + } + + @Test + public void testProcessConsumeMessageDirectly() { + ConsumeMessageDirectlyResultRequestHeader requestHeader = new ConsumeMessageDirectlyResultRequestHeader(); + String remark = "ok"; + int opaque = 123; + RemotingCommand remotingCommand = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, null); + remotingCommand.setOpaque(opaque); + ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); + result.setConsumeResult(CMResult.CR_SUCCESS); + ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(RemotingCommand.class); + CompletableFuture> future = + localProxyRelayService.processConsumeMessageDirectly(ProxyContext.create(), remotingCommand, requestHeader); + future.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, remark, result)); + Mockito.verify(nettyRemotingServerMock, Mockito.times(1)) + .processResponseCommand(Mockito.any(SimpleChannelHandlerContext.class), argumentCaptor.capture()); + RemotingCommand remotingCommand1 = argumentCaptor.getValue(); + assertThat(remotingCommand1.getCode()).isEqualTo(ResponseCode.SUCCESS); + assertThat(remotingCommand1.getRemark()).isEqualTo(remark); + assertThat(remotingCommand1.getBody()).isEqualTo(result.encode()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/ProxyChannelTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/ProxyChannelTest.java new file mode 100644 index 0000000000..167abbe711 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/relay/ProxyChannelTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.relay; + +import io.netty.channel.Channel; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageDecoder; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.protocol.RequestCode; +import org.apache.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; +import org.apache.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import org.apache.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import org.apache.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import org.apache.rocketmq.proxy.service.transaction.TransactionData; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.remoting.protocol.RemotingCommand; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ProxyChannelTest { + + @Mock + private ProxyRelayService proxyRelayService; + + protected abstract static class MockProxyChannel extends ProxyChannel { + + protected MockProxyChannel(ProxyRelayService proxyRelayService, Channel parent, + String remoteAddress, String localAddress) { + super(proxyRelayService, parent, remoteAddress, localAddress); + } + + @Override + public boolean isOpen() { + return false; + } + + @Override + public boolean isActive() { + return false; + } + } + + @Test + public void testWriteAndFlush() throws Exception { + when(this.proxyRelayService.processCheckTransactionState(any(), any(), any(), any())) + .thenReturn(new RelayData<>(mock(TransactionData.class), new CompletableFuture<>())); + + ArgumentCaptor consumeMessageDirectlyArgumentCaptor = + ArgumentCaptor.forClass(ConsumeMessageDirectlyResultRequestHeader.class); + when(this.proxyRelayService.processConsumeMessageDirectly(any(), any(), consumeMessageDirectlyArgumentCaptor.capture())) + .thenReturn(new CompletableFuture<>()); + + ArgumentCaptor getConsumerRunningInfoArgumentCaptor = + ArgumentCaptor.forClass(GetConsumerRunningInfoRequestHeader.class); + when(this.proxyRelayService.processGetConsumerRunningInfo(any(), any(), getConsumerRunningInfoArgumentCaptor.capture())) + .thenReturn(new CompletableFuture<>()); + + CheckTransactionStateRequestHeader checkTransactionStateRequestHeader = new CheckTransactionStateRequestHeader(); + checkTransactionStateRequestHeader.setTransactionId(MessageClientIDSetter.createUniqID()); + RemotingCommand checkTransactionRequest = RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, checkTransactionStateRequestHeader); + MessageExt transactionMessageExt = new MessageExt(); + transactionMessageExt.setTopic("topic"); + transactionMessageExt.setTags("tags"); + transactionMessageExt.setBornHost(RemotingUtil.string2SocketAddress("127.0.0.2:8888")); + transactionMessageExt.setStoreHost(RemotingUtil.string2SocketAddress("127.0.0.1:10911")); + transactionMessageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + transactionMessageExt.setMsgId(MessageClientIDSetter.createUniqID()); + checkTransactionRequest.setBody(MessageDecoder.encode(transactionMessageExt, false)); + + GetConsumerRunningInfoRequestHeader consumerRunningInfoRequestHeader = new GetConsumerRunningInfoRequestHeader(); + consumerRunningInfoRequestHeader.setConsumerGroup("group"); + consumerRunningInfoRequestHeader.setClientId("clientId"); + RemotingCommand consumerRunningInfoRequest = RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, consumerRunningInfoRequestHeader); + + ConsumeMessageDirectlyResultRequestHeader consumeMessageDirectlyResultRequestHeader = new ConsumeMessageDirectlyResultRequestHeader(); + consumeMessageDirectlyResultRequestHeader.setConsumerGroup("group"); + consumeMessageDirectlyResultRequestHeader.setClientId("clientId"); + MessageExt consumeMessageDirectlyMessageExt = new MessageExt(); + consumeMessageDirectlyMessageExt.setTopic("topic"); + consumeMessageDirectlyMessageExt.setTags("tags"); + consumeMessageDirectlyMessageExt.setBornHost(RemotingUtil.string2SocketAddress("127.0.0.2:8888")); + consumeMessageDirectlyMessageExt.setStoreHost(RemotingUtil.string2SocketAddress("127.0.0.1:10911")); + consumeMessageDirectlyMessageExt.setBody(UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8)); + consumeMessageDirectlyMessageExt.setMsgId(MessageClientIDSetter.createUniqID()); + RemotingCommand consumeMessageDirectlyResult = RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, consumeMessageDirectlyResultRequestHeader); + consumeMessageDirectlyResult.setBody(MessageDecoder.encode(consumeMessageDirectlyMessageExt, false)); + + MockProxyChannel channel = new MockProxyChannel(this.proxyRelayService, null, "127.0.0.2:8888", "127.0.0.1:10911") { + @Override + protected CompletableFuture processOtherMessage(Object msg) { + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture processCheckTransaction(CheckTransactionStateRequestHeader header, + MessageExt messageExt, TransactionData transactionData, CompletableFuture> responseFuture) { + assertEquals(checkTransactionStateRequestHeader, header); + assertArrayEquals(transactionMessageExt.getBody(), messageExt.getBody()); + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture processGetConsumerRunningInfo(RemotingCommand command, + GetConsumerRunningInfoRequestHeader header, + CompletableFuture> responseFuture) { + assertEquals(consumerRunningInfoRequestHeader, getConsumerRunningInfoArgumentCaptor.getValue()); + assertEquals(consumerRunningInfoRequestHeader, header); + return CompletableFuture.completedFuture(null); + } + + @Override + protected CompletableFuture processConsumeMessageDirectly(RemotingCommand command, + ConsumeMessageDirectlyResultRequestHeader header, MessageExt messageExt, + CompletableFuture> responseFuture) { + assertEquals(consumeMessageDirectlyResultRequestHeader, consumeMessageDirectlyArgumentCaptor.getValue()); + assertEquals(consumeMessageDirectlyResultRequestHeader, header); + assertArrayEquals(consumeMessageDirectlyMessageExt.getBody(), messageExt.getBody()); + return CompletableFuture.completedFuture(null); + } + }; + + assertTrue(channel.writeAndFlush(checkTransactionRequest).isSuccess()); + assertTrue(channel.writeAndFlush(consumerRunningInfoRequest).isSuccess()); + assertTrue(channel.writeAndFlush(consumeMessageDirectlyResult).isSuccess()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteServiceTest.java new file mode 100644 index 0000000000..2a5d3189eb --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/ClusterTopicRouteServiceTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.net.HostAndPort; +import java.util.List; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.service.BaseServiceTest; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.catchThrowableOfType; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +public class ClusterTopicRouteServiceTest extends BaseServiceTest { + + private ClusterTopicRouteService topicRouteService; + + @Before + public void before() throws Throwable { + super.before(); + this.topicRouteService = new ClusterTopicRouteService(this.mqClientAPIFactory); + + when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(topicRouteData); + when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(ERR_TOPIC), anyLong())).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); + } + + @Test + public void testGetCurrentMessageQueueView() throws Throwable { + MQClientException exception = catchThrowableOfType(() -> this.topicRouteService.getCurrentMessageQueueView(ERR_TOPIC), MQClientException.class); + assertTrue(TopicRouteHelper.isTopicNotExistError(exception)); + assertEquals(1, this.topicRouteService.topicCache.asMap().size()); + + assertNotNull(this.topicRouteService.getCurrentMessageQueueView(TOPIC)); + assertEquals(2, this.topicRouteService.topicCache.asMap().size()); + } + + @Test + public void testGetTopicRouteForProxy() throws Throwable { + List
addressList = Lists.newArrayList(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts("127.0.0.1", 8888))); + ProxyTopicRouteData proxyTopicRouteData = this.topicRouteService.getTopicRouteForProxy(addressList, TOPIC); + + assertEquals(1, proxyTopicRouteData.getBrokerDatas().size()); + assertEquals(addressList, proxyTopicRouteData.getBrokerDatas().get(0).getBrokerAddrs().get(MixAll.MASTER_ID)); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteServiceTest.java new file mode 100644 index 0000000000..709d6cc04c --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteServiceTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.route; + +import com.google.common.net.HostAndPort; +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.rocketmq.broker.BrokerController; +import org.apache.rocketmq.broker.topic.TopicConfigManager; +import org.apache.rocketmq.client.exception.MQClientException; +import org.apache.rocketmq.common.BrokerConfig; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.TopicConfig; +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.common.protocol.ResponseCode; +import org.apache.rocketmq.proxy.common.Address; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.service.BaseServiceTest; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +public class LocalTopicRouteServiceTest extends BaseServiceTest { + + private static final String LOCAL_BROKER_NAME = "localBroker"; + private static final String LOCAL_CLUSTER_NAME = "localCluster"; + private static final String LOCAL_HOST = "127.0.0.2"; + private static final int LOCAL_PORT = 10911; + private static final String LOCAL_ADDR = LOCAL_HOST + ":" + LOCAL_PORT; + @Mock + private BrokerController brokerController; + @Mock + private TopicConfigManager topicConfigManager; + private ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(); + private BrokerConfig brokerConfig = new BrokerConfig(); + private LocalTopicRouteService topicRouteService; + + @Before + public void before() throws Throwable { + super.before(); + this.brokerConfig.setBrokerName(LOCAL_BROKER_NAME); + this.brokerConfig.setBrokerClusterName(LOCAL_CLUSTER_NAME); + + when(this.brokerController.getBrokerAddr()).thenReturn(LOCAL_ADDR); + when(this.brokerController.getBrokerConfig()).thenReturn(this.brokerConfig); + when(this.brokerController.getTopicConfigManager()).thenReturn(this.topicConfigManager); + when(this.topicConfigManager.getTopicConfigTable()).thenReturn(this.topicConfigTable); + + this.topicRouteService = new LocalTopicRouteService(this.brokerController, this.mqClientAPIFactory); + + when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(TOPIC), anyLong())).thenReturn(topicRouteData); + when(this.mqClientAPIExt.getTopicRouteInfoFromNameServer(eq(ERR_TOPIC), anyLong())).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); + } + + @Test + public void testGetCurrentMessageQueueView() throws Throwable { + this.topicConfigTable.put(TOPIC, new TopicConfig(TOPIC, 3, 2, PermName.PERM_WRITE | PermName.PERM_READ)); + MessageQueueView messageQueueView = this.topicRouteService.getCurrentMessageQueueView(TOPIC); + assertEquals(3, messageQueueView.getReadSelector().getQueues().size()); + assertEquals(2, messageQueueView.getWriteSelector().getQueues().size()); + assertEquals(1, messageQueueView.getReadSelector().getBrokerActingQueues().size()); + assertEquals(1, messageQueueView.getWriteSelector().getBrokerActingQueues().size()); + + assertEquals(LOCAL_ADDR, messageQueueView.getReadSelector().selectOne(true).getBrokerAddr()); + assertEquals(LOCAL_BROKER_NAME, messageQueueView.getReadSelector().selectOne(true).getBrokerName()); + assertEquals(messageQueueView.getReadSelector().selectOne(true), messageQueueView.getWriteSelector().selectOne(true)); + } + + @Test + public void testGetTopicRouteForProxy() throws Throwable { + ProxyTopicRouteData proxyTopicRouteData = this.topicRouteService.getTopicRouteForProxy(new ArrayList<>(), TOPIC); + + assertEquals(1, proxyTopicRouteData.getBrokerDatas().size()); + assertEquals( + Lists.newArrayList(new Address(Address.AddressScheme.IPv4, HostAndPort.fromParts( + HostAndPort.fromString(BROKER_ADDR).getHost(), + ConfigurationManager.getProxyConfig().getGrpcServerPort()))), + proxyTopicRouteData.getBrokerDatas().get(0).getBrokerAddrs().get(MixAll.MASTER_ID)); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java new file mode 100644 index 0000000000..e44ed28f4a --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.route; + +import org.apache.rocketmq.common.constant.PermName; +import org.apache.rocketmq.proxy.service.BaseServiceTest; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class MessageQueueSelectorTest extends BaseServiceTest { + + @Test + public void testReadMessageQueue() { + queueData.setPerm(PermName.PERM_READ); + queueData.setReadQueueNums(0); + MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); + assertTrue(messageQueueSelector.getQueues().isEmpty()); + + queueData.setPerm(PermName.PERM_READ); + queueData.setReadQueueNums(3); + messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); + assertEquals(3, messageQueueSelector.getQueues().size()); + assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); + for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { + AddressableMessageQueue messageQueue = messageQueueSelector.getQueues().get(i); + assertEquals(i, messageQueue.getQueueId()); + } + + AddressableMessageQueue brokerQueue = messageQueueSelector.getQueueByBrokerName(BROKER_NAME); + assertEquals(brokerQueue, messageQueueSelector.getBrokerActingQueues().get(0)); + assertEquals(brokerQueue, messageQueueSelector.selectOne(true)); + assertEquals(brokerQueue, messageQueueSelector.selectOneByIndex(3, true)); + + AddressableMessageQueue queue = messageQueueSelector.selectOne(false); + messageQueueSelector.selectOne(false); + messageQueueSelector.selectOne(false); + assertEquals(queue, messageQueueSelector.selectOne(false)); + } + + @Test + public void testWriteMessageQueue() { + queueData.setPerm(PermName.PERM_WRITE); + queueData.setReadQueueNums(0); + MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); + assertTrue(messageQueueSelector.getQueues().isEmpty()); + + queueData.setPerm(PermName.PERM_WRITE); + queueData.setWriteQueueNums(3); + messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); + assertEquals(3, messageQueueSelector.getQueues().size()); + assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); + for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { + AddressableMessageQueue messageQueue = messageQueueSelector.getQueues().get(i); + assertEquals(i, messageQueue.getQueueId()); + } + + AddressableMessageQueue brokerQueue = messageQueueSelector.getQueueByBrokerName(BROKER_NAME); + assertEquals(brokerQueue, messageQueueSelector.getBrokerActingQueues().get(0)); + assertEquals(brokerQueue, messageQueueSelector.selectOne(true)); + assertEquals(brokerQueue, messageQueueSelector.selectOneByIndex(3, true)); + + AddressableMessageQueue queue = messageQueueSelector.selectOne(false); + messageQueueSelector.selectOne(false); + messageQueueSelector.selectOne(false); + assertEquals(queue, messageQueueSelector.selectOne(false)); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java new file mode 100644 index 0000000000..055ab0c0f0 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/AbstractTransactionServiceTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import java.util.List; +import java.util.Random; +import org.apache.rocketmq.common.message.Message; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.common.message.MessageConst; +import org.apache.rocketmq.common.sysflag.MessageSysFlag; +import org.apache.rocketmq.proxy.common.ProxyContext; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class AbstractTransactionServiceTest extends InitConfigAndLoggerTest { + + private static final String BROKER_NAME = "mockBroker"; + private static final String PRODUCER_GROUP = "producerGroup"; + private static final Random RANDOM = new Random(); + + public static class MockAbstractTransactionServiceTest extends AbstractTransactionService { + + @Override + protected String getBrokerNameByAddr(String brokerAddr) { + return BROKER_NAME; + } + + @Override + public void addTransactionSubscription(String group, List topicList) { + + } + + @Override + public void addTransactionSubscription(String group, String topic) { + + } + + @Override + public void replaceTransactionSubscription(String group, List topicList) { + + } + + @Override + public void unSubscribeAllTransactionTopic(String group) { + + } + } + + private TransactionService transactionService; + + @Before + public void before() throws Throwable { + super.before(); + this.transactionService = new MockAbstractTransactionServiceTest(); + } + + @Test + public void testAddAndGenEndHeader() { + Message message = new Message(); + message.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "30"); + String txId = MessageClientIDSetter.createUniqID(); + + TransactionData transactionData = transactionService.addTransactionDataByBrokerName( + BROKER_NAME, + PRODUCER_GROUP, + RANDOM.nextLong(), + RANDOM.nextLong(), + txId, + message + ); + assertNotNull(transactionData); + + EndTransactionRequestData requestData = transactionService.genEndTransactionRequestHeader( + PRODUCER_GROUP, + MessageSysFlag.TRANSACTION_COMMIT_TYPE, + true, + txId, + txId + ); + + assertEquals(BROKER_NAME, requestData.getBrokerName()); + assertEquals(BROKER_NAME, transactionData.getBrokerName()); + assertEquals(transactionData.getCommitLogOffset(), requestData.getRequestHeader().getCommitLogOffset().longValue()); + assertEquals(transactionData.getTranStateTableOffset(), requestData.getRequestHeader().getTranStateTableOffset().longValue()); + + assertNull(transactionService.genEndTransactionRequestHeader( + "group", + MessageSysFlag.TRANSACTION_COMMIT_TYPE, + true, + txId, + txId + )); + } + + @Test + public void testOnSendCheckTransactionStateFailedFailed() { + Message message = new Message(); + message.putUserProperty(MessageConst.PROPERTY_CHECK_IMMUNITY_TIME_IN_SECONDS, "30"); + String txId = MessageClientIDSetter.createUniqID(); + + TransactionData transactionData = transactionService.addTransactionDataByBrokerName( + BROKER_NAME, + PRODUCER_GROUP, + RANDOM.nextLong(), + RANDOM.nextLong(), + txId, + message + ); + transactionService.onSendCheckTransactionStateFailed(ProxyContext.createForInner(this.getClass()), PRODUCER_GROUP, transactionData); + assertNull(transactionService.genEndTransactionRequestHeader( + PRODUCER_GROUP, + MessageSysFlag.TRANSACTION_COMMIT_TYPE, + true, + txId, + txId + )); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java new file mode 100644 index 0000000000..b9706e2af3 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import org.apache.rocketmq.broker.client.ProducerManager; +import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData; +import org.apache.rocketmq.common.protocol.heartbeat.ProducerData; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.service.BaseServiceTest; +import org.apache.rocketmq.proxy.service.route.MessageQueueView; +import org.assertj.core.util.Lists; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; + +public class ClusterTransactionServiceTest extends BaseServiceTest { + + @Mock + private ProducerManager producerManager; + + private ClusterTransactionService clusterTransactionService; + + @Before + public void before() throws Throwable { + super.before(); + this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, null, + this.mqClientAPIFactory); + + MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); + when(this.topicRouteService.getAllMessageQueueView(anyString())) + .thenReturn(messageQueueView); + + when(mqClientAPIFactory.getClient()).thenReturn(mqClientAPIExt); + } + + @Test + public void testAddTransactionSubscription() { + this.clusterTransactionService.addTransactionSubscription(GROUP, TOPIC); + + assertEquals(1, this.clusterTransactionService.getGroupClusterData().size()); + assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster()); + } + + @Test + public void testAddTransactionSubscriptionTopicList() { + this.clusterTransactionService.addTransactionSubscription(GROUP, Lists.newArrayList(TOPIC + 1, TOPIC + 2)); + + assertEquals(1, this.clusterTransactionService.getGroupClusterData().size()); + assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster()); + } + + @Test + public void testReplaceTransactionSubscription() { + this.clusterTransactionService.addTransactionSubscription(GROUP, TOPIC); + + assertEquals(1, this.clusterTransactionService.getGroupClusterData().size()); + assertEquals(CLUSTER_NAME, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster()); + + this.brokerData.setCluster(CLUSTER_NAME + 1); + this.clusterTransactionService.replaceTransactionSubscription(GROUP, Lists.newArrayList(TOPIC + 1)); + assertEquals(1, this.clusterTransactionService.getGroupClusterData().size()); + assertEquals(CLUSTER_NAME + 1, this.clusterTransactionService.getGroupClusterData().get(GROUP).stream().findAny().get().getCluster()); + } + + @Test + public void testUnSubscribeAllTransactionTopic() { + this.clusterTransactionService.addTransactionSubscription(GROUP, TOPIC); + this.clusterTransactionService.unSubscribeAllTransactionTopic(GROUP); + + assertEquals(0, this.clusterTransactionService.getGroupClusterData().size()); + } + + @Test + public void testScanProducerHeartBeat() throws Exception { + ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2); + this.clusterTransactionService.start(); + Set groupSet = new HashSet<>(); + + for (int i = 0; i < 3; i++) { + groupSet.add(GROUP + i); + this.clusterTransactionService.addTransactionSubscription(GROUP + i, TOPIC); + } + + ArgumentCaptor brokerAddrArgumentCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor heartbeatDataArgumentCaptor = ArgumentCaptor.forClass(HeartbeatData.class); + when(mqClientAPIExt.sendHeartbeatOneway( + brokerAddrArgumentCaptor.capture(), + heartbeatDataArgumentCaptor.capture(), + anyLong() + )).thenReturn(CompletableFuture.completedFuture(null)); + + this.clusterTransactionService.scanProducerHeartBeat(); + + await().atMost(Duration.ofSeconds(1)).until(() -> brokerAddrArgumentCaptor.getAllValues().size() == 2); + + assertEquals(Lists.newArrayList(BROKER_ADDR, BROKER_ADDR), brokerAddrArgumentCaptor.getAllValues()); + List heartbeatDataList = heartbeatDataArgumentCaptor.getAllValues(); + for (ProducerData producerData : heartbeatDataList.get(0).getProducerDataSet()) { + groupSet.remove(producerData.getGroupName()); + } + + for (ProducerData producerData : heartbeatDataList.get(1).getProducerDataSet()) { + groupSet.remove(producerData.getGroupName()); + } + + assertTrue(groupSet.isEmpty()); + } +} \ No newline at end of file diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java new file mode 100644 index 0000000000..2d03ab6af4 --- /dev/null +++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/TransactionDataManagerTest.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.proxy.service.transaction; + +import java.time.Duration; +import java.util.Random; +import org.apache.commons.lang3.time.StopWatch; +import org.apache.rocketmq.common.message.MessageClientIDSetter; +import org.apache.rocketmq.proxy.config.InitConfigAndLoggerTest; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import static org.awaitility.Awaitility.await; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +public class TransactionDataManagerTest extends InitConfigAndLoggerTest { + private static final String PRODUCER_GROUP = "producerGroup"; + private static final Random RANDOM = new Random(); + private TransactionDataManager transactionDataManager; + + @Before + public void before() throws Throwable { + super.before(); + this.transactionDataManager = new TransactionDataManager(); + } + + @After + public void after() { + super.after(); + } + + @Test + public void testAddAndRemove() { + TransactionData transactionData1 = createTransactionData(); + TransactionData transactionData2 = createTransactionData(transactionData1.getTransactionId()); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData1); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData2); + + assertEquals(1, this.transactionDataManager.transactionIdDataMap.size()); + assertEquals(2, this.transactionDataManager.transactionIdDataMap.get( + transactionDataManager.buildKey(PRODUCER_GROUP, transactionData1.getTransactionId())).size()); + + this.transactionDataManager.removeTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData1); + assertEquals(1, this.transactionDataManager.transactionIdDataMap.size()); + this.transactionDataManager.removeTransactionData(PRODUCER_GROUP, transactionData1.getTransactionId(), transactionData2); + assertEquals(0, this.transactionDataManager.transactionIdDataMap.size()); + } + + @Test + public void testPollFirst() { + String txId = MessageClientIDSetter.createUniqID(); + TransactionData transactionData1 = createTransactionData(txId, System.currentTimeMillis() - Duration.ofMinutes(2).toMillis()); + TransactionData transactionData2 = createTransactionData(txId); + + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, transactionData1); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, transactionData2); + + TransactionData resTransactionData = this.transactionDataManager.pollFirstNoExpireTransactionData(PRODUCER_GROUP, txId); + assertSame(transactionData2, resTransactionData); + assertTrue(this.transactionDataManager.transactionIdDataMap.isEmpty()); + } + + @Test + public void testCleanExpire() { + String txId = MessageClientIDSetter.createUniqID(); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(100).toMillis())); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(500).toMillis())); + + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, MessageClientIDSetter.createUniqID(), + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(1000).toMillis())); + + await().atMost(Duration.ofSeconds(2)).until(() -> { + this.transactionDataManager.cleanExpireTransactionData(); + return this.transactionDataManager.transactionIdDataMap.isEmpty(); + }); + } + + @Test + public void testWaitTransactionDataClear() throws InterruptedException { + String txId = MessageClientIDSetter.createUniqID(); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(100).toMillis())); + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, txId, + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(500).toMillis())); + + this.transactionDataManager.addTransactionData(PRODUCER_GROUP, MessageClientIDSetter.createUniqID(), + createTransactionData(txId, System.currentTimeMillis(), Duration.ofMillis(1000).toMillis())); + + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + this.transactionDataManager.waitTransactionDataClear(); + stopWatch.stop(); + assertTrue(Math.abs(stopWatch.getTime() - 1000) <= 50); + } + + private static TransactionData createTransactionData() { + return createTransactionData(MessageClientIDSetter.createUniqID()); + } + + private static TransactionData createTransactionData(String txId) { + return createTransactionData(txId, System.currentTimeMillis()); + } + + private static TransactionData createTransactionData(String txId, long checkTimestamp) { + return createTransactionData(txId, checkTimestamp, Duration.ofMinutes(1).toMillis()); + } + + private static TransactionData createTransactionData(String txId, long checkTimestamp, long checkImmunityTime) { + return new TransactionData( + "brokerName", + RANDOM.nextLong(), + RANDOM.nextLong(), + txId, + checkTimestamp, + checkImmunityTime + ); + } +} \ No newline at end of file diff --git a/proxy/src/test/resources/rmq-proxy-home/conf/broker.conf b/proxy/src/test/resources/rmq-proxy-home/conf/broker.conf new file mode 100644 index 0000000000..0c0b28b7b8 --- /dev/null +++ b/proxy/src/test/resources/rmq-proxy-home/conf/broker.conf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName = DefaultCluster +brokerName = broker-a +brokerId = 0 +deleteWhen = 04 +fileReservedTime = 48 +brokerRole = ASYNC_MASTER +flushDiskType = ASYNC_FLUSH diff --git a/proxy/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml b/proxy/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml new file mode 100644 index 0000000000..74829684e8 --- /dev/null +++ b/proxy/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml @@ -0,0 +1,420 @@ + + + + + + + ${user.home}/logs/rocketmqlogs/proxy.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/proxy.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/proxy_watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/proxy_watermark.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker_default.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker.%i.log.gz + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/protection.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/protection.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/watermark.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/store.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/store.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/remoting.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/remoting.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/storeerror.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/storeerror.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/transaction.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/transaction.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/lock.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/lock.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/filter.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/filter.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/stats.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/stats.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/commercial.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/commercial.%i.log.gz + 1 + 10 + + + 500MB + + + + + ${user.home}/logs/rocketmqlogs/pop.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/pop.%i.log + + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proxy/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json b/proxy/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json new file mode 100644 index 0000000000..f0873e2a35 --- /dev/null +++ b/proxy/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json @@ -0,0 +1,3 @@ +{ + "proxyMode": "cluster" +} \ No newline at end of file diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java index 62d043e679..41edb9620b 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java @@ -38,9 +38,6 @@ public class NettyClientConfig { private boolean clientPooledByteBufAllocatorEnable = false; private boolean clientCloseSocketIfTimeout = NettySystemConfig.clientCloseSocketIfTimeout; - private boolean preferredDirectByteBuffer = false; - private boolean defaultEventExecutorGroupEnable = true; - private boolean useTLS; private int writeBufferHighWaterMark = NettySystemConfig.writeBufferHighWaterMark; @@ -169,4 +166,11 @@ public class NettyClientConfig { this.disableCallbackExecutor = disableCallbackExecutor; } + public boolean isDisableNettyWorkerGroup() { + return disableNettyWorkerGroup; + } + + public void setDisableNettyWorkerGroup(boolean disableNettyWorkerGroup) { + this.disableNettyWorkerGroup = disableNettyWorkerGroup; + } } diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java index ce3a157fa5..08e1fec9cf 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java @@ -210,8 +210,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti LOGGER.warn("Connections are insecure as SSLContext is null!"); } } - pipeline.addLast( - defaultEventExecutorGroup, + ch.pipeline().addLast( + nettyClientConfig.isDisableNettyWorkerGroup() ? null : defaultEventExecutorGroup, new NettyEncoder(), new NettyDecoder(), new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()), diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java index 720beb030c..631ad75bfe 100644 --- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java @@ -110,7 +110,7 @@ public class RemotingCommand { return cmd; } - private static void setCmdVersion(RemotingCommand cmd) { + protected static void setCmdVersion(RemotingCommand cmd) { if (configVersion >= 0) { cmd.setVersion(configVersion); } else { diff --git a/test/pom.xml b/test/pom.xml index 8f902c0885..312504a382 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -32,6 +32,18 @@ log4j log4j + + ${project.groupId} + rocketmq-proto + + + com.google.protobuf + protobuf-java-util + + + ${project.groupId} + rocketmq-proxy + ${project.groupId} rocketmq-broker @@ -68,6 +80,15 @@ ${project.groupId} rocketmq-tools + + io.grpc + grpc-testing + test + + + org.awaitility + awaitility + diff --git a/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java index d376cd0a12..2e449dee7b 100644 --- a/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java +++ b/test/src/main/java/org/apache/rocketmq/test/util/MQAdminTestUtils.java @@ -17,6 +17,7 @@ package org.apache.rocketmq.test.util; +import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.PosixParser; @@ -50,19 +51,19 @@ import java.util.UUID; import java.util.concurrent.ForkJoinPool; import static org.apache.rocketmq.common.statictopic.TopicQueueMappingUtils.getMappingDetailFromConfig; +import static org.awaitility.Awaitility.await; public class MQAdminTestUtils { private static Logger log = Logger.getLogger(MQAdminTestUtils.class); public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, int queueNum, Map attributes) { - int defaultWaitTime = 5; + int defaultWaitTime = 30; return createTopic(nameSrvAddr, clusterName, topic, queueNum, attributes, defaultWaitTime); } public static boolean createTopic(String nameSrvAddr, String clusterName, String topic, int queueNum, Map attributes, int waitTimeSec) { - boolean createResult = false; DefaultMQAdminExt mqAdminExt = new DefaultMQAdminExt(); mqAdminExt.setInstanceName(UUID.randomUUID().toString()); mqAdminExt.setNamesrvAddr(nameSrvAddr); @@ -72,19 +73,9 @@ public class MQAdminTestUtils { } catch (Exception e) { } - long startTime = System.currentTimeMillis(); - while (!createResult) { - createResult = checkTopicExist(mqAdminExt, topic); - if (System.currentTimeMillis() - startTime < waitTimeSec * 1000) { - TestUtils.waitForMoment(100); - } else { - log.error(String.format("timeout,but create topic[%s] failed!", topic)); - break; - } - } - + await().atMost(waitTimeSec, TimeUnit.SECONDS).until(() -> checkTopicExist(mqAdminExt, topic)); ForkJoinPool.commonPool().execute(mqAdminExt::shutdown); - return createResult; + return true; } private static boolean checkTopicExist(DefaultMQAdminExt mqAdminExt, String topic) { diff --git a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java index 4e29c84c6b..0f26f2042c 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/BaseConf.java @@ -34,10 +34,11 @@ import org.apache.rocketmq.client.consumer.MQPushConsumer; import org.apache.rocketmq.client.producer.MQProducer; import org.apache.rocketmq.client.producer.TransactionListener; import org.apache.rocketmq.common.MQVersion; +import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.attribute.TopicMessageType; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.namesrv.NamesrvController; import org.apache.rocketmq.remoting.protocol.RemotingCommand; -import org.apache.rocketmq.common.attribute.CQType; import org.apache.rocketmq.test.client.rmq.RMQAsyncSendProducer; import org.apache.rocketmq.test.client.rmq.RMQNormalConsumer; import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; @@ -92,7 +93,8 @@ public class BaseConf { } public BaseConf() { - + // Add waitBrokerRegistered to BaseConf constructor to make it default for all subclasses. + waitBrokerRegistered(nsAddr, clusterName, brokerNum); } // This method can't be placed in the static block of BaseConf, which seems to lead to a strange dead lock. @@ -101,9 +103,13 @@ public class BaseConf { mqAdminExt.setNamesrvAddr(nsAddr); try { mqAdminExt.start(); - Thread.sleep(10000); await().atMost(30, TimeUnit.SECONDS).until(() -> { - List brokerDatas = mqAdminExt.examineTopicRouteInfo(clusterName).getBrokerDatas(); + List brokerDatas; + try { + brokerDatas = mqAdminExt.examineTopicRouteInfo(clusterName).getBrokerDatas(); + } catch (Exception e) { + return false; + } return brokerDatas.size() == expectedBrokerNum; }); for (BrokerController brokerController: brokerControllerList) { @@ -140,11 +146,41 @@ public class BaseConf { return initTopicWithName(topic); } + public static String initTopic(TopicMessageType topicMessageType) { + String topic = MQRandomUtils.getRandomTopic(); + return initTopicWithName(topic, topicMessageType); + } + + public static String initTopicOnSampleTopicBroker(String sampleTopic) { + String topic = MQRandomUtils.getRandomTopic(); + return initTopicOnSampleTopicBroker(topic, sampleTopic); + } + + public static String initTopicOnSampleTopicBroker(String sampleTopic, TopicMessageType topicMessageType) { + String topic = MQRandomUtils.getRandomTopic(); + return initTopicOnSampleTopicBroker(topic, sampleTopic, topicMessageType); + } + public static String initTopicWithName(String topicName) { IntegrationTestBase.initTopic(topicName, nsAddr, clusterName, CQType.SimpleCQ); return topicName; } + public static String initTopicWithName(String topicName, TopicMessageType topicMessageType) { + IntegrationTestBase.initTopic(topicName, nsAddr, clusterName, topicMessageType); + return topicName; + } + + public static String initTopicOnSampleTopicBroker(String topicName, String sampleTopic) { + IntegrationTestBase.initTopic(topicName, nsAddr, sampleTopic, CQType.SimpleCQ); + return topicName; + } + + public static String initTopicOnSampleTopicBroker(String topicName, String sampleTopic, TopicMessageType topicMessageType) { + IntegrationTestBase.initTopic(topicName, nsAddr, sampleTopic, topicMessageType); + return topicName; + } + public static String initConsumerGroup() { String group = MQRandomUtils.getRandomConsumerGroup(); return initConsumerGroup(group); diff --git a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java index 398bd1943c..9b09d16320 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/IntegrationTestBase.java @@ -26,12 +26,14 @@ import java.util.Map; import java.util.Objects; import java.util.Random; import java.util.UUID; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.TopicAttributes; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.attribute.CQType; +import org.apache.rocketmq.common.attribute.TopicMessageType; import org.apache.rocketmq.common.namesrv.NamesrvConfig; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; @@ -40,7 +42,6 @@ import org.apache.rocketmq.remoting.netty.NettyClientConfig; import org.apache.rocketmq.remoting.netty.NettyServerConfig; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.test.util.MQAdminTestUtils; -import org.apache.rocketmq.test.util.TestUtils; public class IntegrationTestBase { public static InternalLogger logger = InternalLoggerFactory.getLogger(IntegrationTestBase.class); @@ -51,7 +52,7 @@ public class IntegrationTestBase { protected static final List TMPE_FILES = new ArrayList<>(); protected static final List BROKER_CONTROLLERS = new ArrayList<>(); protected static final List NAMESRV_CONTROLLERS = new ArrayList<>(); - protected static int topicCreateTime = 30 * 1000; + protected static int topicCreateTime = (int) TimeUnit.SECONDS.toSeconds(30); public static volatile int COMMIT_LOG_SIZE = 1024 * 1024 * 100; protected static final int INDEX_NUM = 1000; @@ -169,32 +170,28 @@ public class IntegrationTestBase { } public static boolean initTopic(String topic, String nsAddr, String clusterName, int queueNumbers, CQType cqType) { - long startTime = System.currentTimeMillis(); + return initTopic(topic, nsAddr, clusterName, queueNumbers, cqType, TopicMessageType.NORMAL); + } + + public static boolean initTopic(String topic, String nsAddr, String clusterName, int queueNumbers, CQType cqType, TopicMessageType topicMessageType) { boolean createResult; - - while (true) { - Map attributes = new HashMap<>(); - if (!Objects.equals(CQType.SimpleCQ, cqType)) { - attributes.put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), cqType.toString()); - } - createResult = MQAdminTestUtils.createTopic(nsAddr, clusterName, topic, queueNumbers, attributes); - if (createResult) { - break; - } else if (System.currentTimeMillis() - startTime > topicCreateTime) { - Truth.assertWithMessage(String.format("topic[%s] is created failed after:%d ms", topic, - System.currentTimeMillis() - startTime)).fail(); - break; - } else { - TestUtils.waitForMoment(500); - continue; - } + Map attributes = new HashMap<>(); + if (!Objects.equals(CQType.SimpleCQ, cqType)) { + attributes.put("+" + TopicAttributes.QUEUE_TYPE_ATTRIBUTE.getName(), cqType.toString()); } - + if (!Objects.equals(TopicMessageType.NORMAL, topicMessageType)) { + attributes.put("+" + TopicAttributes.TOPIC_MESSAGE_TYPE_ATTRIBUTE.getName(), topicMessageType.toString()); + } + createResult = MQAdminTestUtils.createTopic(nsAddr, clusterName, topic, queueNumbers, attributes, topicCreateTime); return createResult; } public static boolean initTopic(String topic, String nsAddr, String clusterName, CQType cqType) { - return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS, cqType); + return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS, cqType, TopicMessageType.NORMAL); + } + + public static boolean initTopic(String topic, String nsAddr, String clusterName, TopicMessageType topicMessageType) { + return initTopic(topic, nsAddr, clusterName, BaseConf.QUEUE_NUMBERS, CQType.SimpleCQ, topicMessageType); } public static void deleteFile(File file) { diff --git a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java index 0e1bf2658b..7b9b9e5f33 100644 --- a/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/base/dledger/DLedgerProduceAndConsumeIT.java @@ -75,7 +75,7 @@ public class DLedgerProduceAndConsumeIT { BrokerConfig brokerConfig = buildBrokerConfig(cluster, brokerName); MessageStoreConfig storeConfig = buildStoreConfig(brokerName, peers, selfId); BrokerController brokerController = IntegrationTestBase.createAndStartBroker(storeConfig, brokerConfig); - Thread.sleep(3000); + BaseConf.waitBrokerRegistered(BaseConf.nsAddr, brokerConfig.getBrokerName(), 1); Assert.assertEquals(BrokerRole.SYNC_MASTER, storeConfig.getBrokerRole()); diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java new file mode 100644 index 0000000000..88f4e8ba25 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/ClusterGrpcIT.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.grpc.v2; + +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteResponse; +import java.time.Duration; +import java.util.Map; +import org.apache.rocketmq.common.protocol.route.BrokerData; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication; +import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.apache.rocketmq.test.util.MQAdminTestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +import static org.awaitility.Awaitility.await; + +@FixMethodOrder(value = MethodSorters.NAME_ASCENDING) +public class ClusterGrpcIT extends GrpcBaseIT { + + private MessagingProcessor messagingProcessor; + private GrpcMessagingApplication grpcMessagingApplication; + + @Before + public void setUp() throws Exception { + super.setUp(); + ConfigurationManager.getProxyConfig().setTransactionHeartbeatPeriodSecond(3); + messagingProcessor = DefaultMessagingProcessor.createForClusterMode(); + messagingProcessor.start(); + grpcMessagingApplication = GrpcMessagingApplication.create(messagingProcessor); + grpcMessagingApplication.start(); + setUpServer(grpcMessagingApplication, ConfigurationManager.getProxyConfig().getGrpcServerPort(), true); + + await().atMost(Duration.ofSeconds(40)).until(() -> { + Map brokerDataMap = MQAdminTestUtils.getCluster(nsAddr).getBrokerAddrTable(); + return brokerDataMap.size() == brokerNum; + }); + } + + @After + public void tearDown() throws Exception { + messagingProcessor.shutdown(); + grpcMessagingApplication.shutdown(); + shutdown(); + } + + @Test + public void testQueryRoute() throws Exception { + String topic = initTopic(); + + QueryRouteResponse response = blockingStub.queryRoute(buildQueryRouteRequest(topic)); + assertQueryRoute(response, brokerNum * DEFAULT_QUEUE_NUMS); + } + + @Test + public void testQueryAssignment() throws Exception { + String topic = initTopic(); + String group = "group"; + + QueryAssignmentResponse response = blockingStub.queryAssignment(buildQueryAssignmentRequest(topic, group)); + + assertQueryAssignment(response, brokerNum); + } + + @Test + public void testTransactionCheckThenCommit() { + super.testTransactionCheckThenCommit(); + } + + @Test + public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { + super.testSimpleConsumerSendAndRecvDelayMessage(); + } + + @Test + public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { + super.testSimpleConsumerSendAndRecvBigMessage(); + } + + @Test + public void testSimpleConsumerSendAndRecv() throws Exception { + super.testSimpleConsumerSendAndRecv(); + } + + @Test + public void testSimpleConsumerToDLQ() throws Exception { + super.testSimpleConsumerToDLQ(); + } + + @Test + public void testConsumeOrderly() throws Exception { + super.testConsumeOrderly(); + } +} diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java new file mode 100644 index 0000000000..35f01920dc --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/GrpcBaseIT.java @@ -0,0 +1,805 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.grpc.v2; + +import apache.rocketmq.v2.AckMessageEntry; +import apache.rocketmq.v2.AckMessageRequest; +import apache.rocketmq.v2.AckMessageResponse; +import apache.rocketmq.v2.AckMessageResultEntry; +import apache.rocketmq.v2.Address; +import apache.rocketmq.v2.AddressScheme; +import apache.rocketmq.v2.ChangeInvisibleDurationRequest; +import apache.rocketmq.v2.ChangeInvisibleDurationResponse; +import apache.rocketmq.v2.ClientType; +import apache.rocketmq.v2.Code; +import apache.rocketmq.v2.Encoding; +import apache.rocketmq.v2.EndTransactionRequest; +import apache.rocketmq.v2.EndTransactionResponse; +import apache.rocketmq.v2.Endpoints; +import apache.rocketmq.v2.HeartbeatRequest; +import apache.rocketmq.v2.Message; +import apache.rocketmq.v2.MessageQueue; +import apache.rocketmq.v2.MessageType; +import apache.rocketmq.v2.MessagingServiceGrpc; +import apache.rocketmq.v2.Publishing; +import apache.rocketmq.v2.QueryAssignmentRequest; +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteRequest; +import apache.rocketmq.v2.QueryRouteResponse; +import apache.rocketmq.v2.ReceiveMessageRequest; +import apache.rocketmq.v2.ReceiveMessageResponse; +import apache.rocketmq.v2.RecoverOrphanedTransactionCommand; +import apache.rocketmq.v2.Resource; +import apache.rocketmq.v2.RetryPolicy; +import apache.rocketmq.v2.SendMessageRequest; +import apache.rocketmq.v2.SendMessageResponse; +import apache.rocketmq.v2.Settings; +import apache.rocketmq.v2.Subscription; +import apache.rocketmq.v2.SystemProperties; +import apache.rocketmq.v2.TelemetryCommand; +import apache.rocketmq.v2.TransactionResolution; +import apache.rocketmq.v2.TransactionSource; +import com.google.protobuf.ByteString; +import com.google.protobuf.Duration; +import com.google.protobuf.util.Durations; +import com.google.protobuf.util.Timestamps; +import io.grpc.Channel; +import io.grpc.Metadata; +import io.grpc.ServerInterceptors; +import io.grpc.ServerServiceDefinition; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.ApplicationProtocolConfig; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslProvider; +import io.grpc.stub.MetadataUtils; +import io.grpc.stub.StreamObserver; +import io.grpc.testing.GrpcCleanupRule; +import io.netty.handler.ssl.ApplicationProtocolNames; +import io.netty.handler.ssl.util.InsecureTrustManagerFactory; +import io.netty.handler.ssl.util.SelfSignedCertificate; +import java.io.IOException; +import java.net.URL; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import javax.net.ssl.SSLException; +import org.apache.commons.lang3.StringUtils; +import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; +import org.apache.rocketmq.client.consumer.PullResult; +import org.apache.rocketmq.client.consumer.PullStatus; +import org.apache.rocketmq.common.MixAll; +import org.apache.rocketmq.common.attribute.TopicMessageType; +import org.apache.rocketmq.common.message.MessageExt; +import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor; +import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor; +import org.apache.rocketmq.proxy.grpc.interceptor.InterceptorConstants; +import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +import org.apache.rocketmq.remoting.common.RemotingUtil; +import org.apache.rocketmq.test.base.BaseConf; +import org.apache.rocketmq.test.util.MQRandomUtils; +import org.apache.rocketmq.test.util.RandomUtils; +import org.junit.Rule; + +import static org.apache.rocketmq.common.message.MessageClientIDSetter.createUniqID; +import static org.apache.rocketmq.proxy.config.ConfigurationManager.RMQ_PROXY_HOME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +public class GrpcBaseIT extends BaseConf { + + protected final int port = 8082; + /** + * This rule manages automatic graceful shutdown for the registered servers and channels at the end of test. + */ + @Rule + public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule(); + + protected MessagingServiceGrpc.MessagingServiceBlockingStub blockingStub; + protected MessagingServiceGrpc.MessagingServiceStub stub; + protected final Metadata header = new Metadata(); + + protected static final int DEFAULT_QUEUE_NUMS = 8; + + public void setUp() throws Exception { + brokerController1.getBrokerConfig().setTransactionCheckInterval(3 * 1000); + brokerController2.getBrokerConfig().setTransactionCheckInterval(3 * 1000); + brokerController3.getBrokerConfig().setTransactionCheckInterval(3 * 1000); + + header.put(InterceptorConstants.CLIENT_ID, "client-id" + UUID.randomUUID()); + header.put(InterceptorConstants.LANGUAGE, "JAVA"); + + String mockProxyHome = "/mock/rmq/proxy/home"; + URL mockProxyHomeURL = getClass().getClassLoader().getResource("rmq-proxy-home"); + if (mockProxyHomeURL != null) { + mockProxyHome = mockProxyHomeURL.toURI().getPath(); + } + System.setProperty(RMQ_PROXY_HOME, mockProxyHome); + ConfigurationManager.initEnv(); + ConfigurationManager.intConfig(); + ConfigurationManager.getProxyConfig().setGrpcServerPort(port); + ConfigurationManager.getProxyConfig().setNameSrvAddr(nsAddr); + // Set LongPollingReserveTimeInMillis to 500ms to reserve more time for IT + ConfigurationManager.getProxyConfig().setLongPollingReserveTimeInMillis(500); + ConfigurationManager.getProxyConfig().setRocketMQClusterName(brokerController1.getBrokerConfig().getBrokerClusterName()); + ConfigurationManager.getProxyConfig().setMinInvisibleTimeMillsForRecv(3); + + blockingStub = createBlockingStub(createChannel(ConfigurationManager.getProxyConfig().getGrpcServerPort())); + stub = createStub(createChannel(ConfigurationManager.getProxyConfig().getGrpcServerPort())); + } + + protected MessagingServiceGrpc.MessagingServiceStub createStub(Channel channel) { + MessagingServiceGrpc.MessagingServiceStub stub = MessagingServiceGrpc.newStub(channel); + return stub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header)); + } + + protected MessagingServiceGrpc.MessagingServiceBlockingStub createBlockingStub(Channel channel) { + MessagingServiceGrpc.MessagingServiceBlockingStub stub = MessagingServiceGrpc.newBlockingStub(channel); + return stub.withInterceptors(MetadataUtils.newAttachHeadersInterceptor(header)); + } + + protected CompletableFuture sendClientSettings(MessagingServiceGrpc.MessagingServiceStub stub, + Settings clientSettings) { + CompletableFuture future = new CompletableFuture<>(); + StreamObserver requestStreamObserver = stub.telemetry(new DefaultTelemetryCommandStreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + TelemetryCommand.CommandCase commandCase = value.getCommandCase(); + if (TelemetryCommand.CommandCase.SETTINGS.equals(commandCase)) { + future.complete(value.getSettings()); + } + } + }); + requestStreamObserver.onNext(TelemetryCommand.newBuilder() + .setSettings(clientSettings) + .build()); + future.whenComplete((settings, throwable) -> requestStreamObserver.onCompleted()); + return future; + } + + protected void setUpServer(MessagingServiceGrpc.MessagingServiceImplBase serverImpl, + int port, boolean enableInterceptor) throws IOException, CertificateException { + SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate(); + ServerServiceDefinition serviceDefinition = ServerInterceptors.intercept(serverImpl); + if (enableInterceptor) { + serviceDefinition = ServerInterceptors.intercept(serverImpl, new ContextInterceptor(), new HeaderInterceptor()); + } + // Create a server, add service, start, and register for automatic graceful shutdown. + grpcCleanup.register(NettyServerBuilder.forPort(port) + .directExecutor() + .addService(serviceDefinition) + .useTransportSecurity(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey()) + .build() + .start()); + } + + protected Channel createChannel(int port) throws SSLException { + return grpcCleanup.register(NettyChannelBuilder.forAddress("127.0.0.1", port) + .directExecutor() + .sslContext(SslContextBuilder + .forClient() + .sslProvider(SslProvider.OPENSSL) + .trustManager(InsecureTrustManagerFactory.INSTANCE) + .applicationProtocolConfig(new ApplicationProtocolConfig( + ApplicationProtocolConfig.Protocol.ALPN, + ApplicationProtocolConfig.SelectorFailureBehavior.NO_ADVERTISE, + ApplicationProtocolConfig.SelectedListenerFailureBehavior.ACCEPT, + ApplicationProtocolNames.HTTP_2)) + .build() + ) + .build()); + } + + public void testTransactionCheckThenCommit() { + String topic = initTopicOnSampleTopicBroker(broker1Name, TopicMessageType.TRANSACTION); + String group = MQRandomUtils.getRandomConsumerGroup(); + + AtomicReference telemetryCommandRef = new AtomicReference<>(null); + StreamObserver requestStreamObserver = stub.telemetry(new DefaultTelemetryCommandStreamObserver() { + @Override + public void onNext(TelemetryCommand value) { + telemetryCommandRef.set(value); + } + }); + + try { + requestStreamObserver.onNext(TelemetryCommand.newBuilder() + .setSettings(buildPushConsumerClientSettings(group)) + .build()); + await().atMost(java.time.Duration.ofSeconds(3)).until(() -> { + if (telemetryCommandRef.get() == null) { + return false; + } + if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) { + return false; + } + return telemetryCommandRef.get() != null; + }); + telemetryCommandRef.set(null); + // init consumer offset + receiveMessage(blockingStub, topic, group, 1); + + requestStreamObserver.onNext(TelemetryCommand.newBuilder() + .setSettings(buildProducerClientSettings(topic)) + .build()); + blockingStub.heartbeat(buildHeartbeatRequest(group)); + await().atMost(java.time.Duration.ofSeconds(3)).until(() -> { + if (telemetryCommandRef.get() == null) { + blockingStub.heartbeat(buildHeartbeatRequest(group)); + return false; + } + if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.SETTINGS) { + blockingStub.heartbeat(buildHeartbeatRequest(group)); + return false; + } + return telemetryCommandRef.get() != null; + }); + telemetryCommandRef.set(null); + + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(buildTransactionSendMessageRequest(topic, messageId)); + assertSendMessage(sendResponse, messageId); + + await().atMost(java.time.Duration.ofMinutes(2)).until(() -> { + if (telemetryCommandRef.get() == null) { + blockingStub.heartbeat(buildHeartbeatRequest(group)); + return false; + } + if (telemetryCommandRef.get().getCommandCase() != TelemetryCommand.CommandCase.RECOVER_ORPHANED_TRANSACTION_COMMAND) { + blockingStub.heartbeat(buildHeartbeatRequest(group)); + return false; + } + return telemetryCommandRef.get() != null; + }); + RecoverOrphanedTransactionCommand recoverOrphanedTransactionCommand = telemetryCommandRef.get().getRecoverOrphanedTransactionCommand(); + assertRecoverOrphanedTransactionCommand(recoverOrphanedTransactionCommand, messageId); + + EndTransactionResponse endTransactionResponse = blockingStub.endTransaction( + buildEndTransactionRequest(topic, messageId, recoverOrphanedTransactionCommand.getTransactionId(), TransactionResolution.COMMIT)); + assertEndTransactionResponse(endTransactionResponse); + + requestStreamObserver.onNext(TelemetryCommand.newBuilder() + .setSettings(buildPushConsumerClientSettings(group)) + .build()); + + await().atMost(java.time.Duration.ofSeconds(30)).until(() -> { + List retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group)); + if (retryMessageList.isEmpty()) { + return false; + } + return retryMessageList.get(0).getSystemProperties() + .getMessageId().equals(messageId); + }); + } finally { + requestStreamObserver.onCompleted(); + } + } + + public HeartbeatRequest buildHeartbeatRequest(String group) { + return HeartbeatRequest.newBuilder() + .setGroup(Resource.newBuilder() + .setName(group) + .build()) + .build(); + } + + public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { + String topic = initTopicOnSampleTopicBroker(broker1Name, TopicMessageType.DELAY); + String group = MQRandomUtils.getRandomConsumerGroup(); + long delayTime = TimeUnit.SECONDS.toMillis(5); + + // init consumer offset + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBodyEncoding(Encoding.GZIP) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .setDeliveryTimestamp(Timestamps.fromMillis(System.currentTimeMillis() + delayTime)) + .build()) + .setBody(ByteString.copyFromUtf8("hello")) + .build()) + .build()); + long sendTime = System.currentTimeMillis(); + assertSendMessage(sendResponse, messageId); + + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + + AtomicLong recvTime = new AtomicLong(); + AtomicReference recvMessage = new AtomicReference<>(); + await().atMost(java.time.Duration.ofSeconds(10)).until(() -> { + List messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group)); + if (messageList.isEmpty()) { + return false; + } + recvTime.set(System.currentTimeMillis()); + recvMessage.set(messageList.get(0)); + return messageList.get(0).getSystemProperties().getMessageId().equals(messageId); + }); + + assertThat(Math.abs(recvTime.get() - sendTime - delayTime) < 2 * 1000).isTrue(); + } + + public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { + String topic = initTopicOnSampleTopicBroker(broker1Name); + String group = MQRandomUtils.getRandomConsumerGroup(); + + int bodySize = 4 * 1024; + + // init consumer offset + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendBigMessageRequest(topic, messageId, bodySize)); + assertSendMessage(sendResponse, messageId); + + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + + Message message = assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId); + assertThat(message.getSystemProperties().getBodyEncoding()).isEqualTo(Encoding.GZIP); + assertThat(message.getBody().size()).isEqualTo(bodySize); + } + + public void testSimpleConsumerSendAndRecv() throws Exception { + String topic = initTopicOnSampleTopicBroker(broker1Name); + String group = MQRandomUtils.getRandomConsumerGroup(); + + // init consumer offset + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId)); + assertSendMessage(sendResponse, messageId); + + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + + Message message = assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId); + + String receiptHandle = message.getSystemProperties().getReceiptHandle(); + ChangeInvisibleDurationResponse changeResponse = blockingStub.changeInvisibleDuration(buildChangeInvisibleDurationRequest(topic, group, receiptHandle, 5)); + assertChangeInvisibleDurationResponse(changeResponse, receiptHandle); + + List ackHandles = new ArrayList<>(); + ackHandles.add(changeResponse.getReceiptHandle()); + + await().atMost(java.time.Duration.ofSeconds(20)).until(() -> { + List retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group)); + if (retryMessageList.isEmpty()) { + return false; + } + if (retryMessageList.get(0).getSystemProperties() + .getMessageId().equals(messageId)) { + ackHandles.add(retryMessageList.get(0).getSystemProperties().getReceiptHandle()); + return true; + } + return false; + }); + + assertThat(ackHandles.size()).isEqualTo(2); + AckMessageResponse ackMessageResponse = blockingStub.ackMessage(buildAckMessageRequest(topic, group, + AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(ackHandles.get(0)).build(), + AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(ackHandles.get(1)).build())); + assertThat(ackMessageResponse.getStatus().getCode()).isEqualTo(Code.MULTIPLE_RESULTS); + int okNum = 0; + int expireNum = 0; + for (AckMessageResultEntry entry : ackMessageResponse.getEntriesList()) { + if (entry.getStatus().getCode().equals(Code.OK)) { + okNum++; + } else if (entry.getStatus().getCode().equals(Code.INVALID_RECEIPT_HANDLE)) { + expireNum++; + } + } + assertThat(okNum).isEqualTo(1); + assertThat(expireNum).isEqualTo(1); + } + + public void testSimpleConsumerToDLQ() throws Exception { + String topic = initTopicOnSampleTopicBroker(broker1Name); + String group = MQRandomUtils.getRandomConsumerGroup(); + int maxDeliveryAttempts = 2; + + SubscriptionGroupConfig groupConfig = brokerController1.getSubscriptionGroupManager().findSubscriptionGroupConfig(group); + groupConfig.setRetryMaxTimes(maxDeliveryAttempts - 1); + brokerController1.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + brokerController2.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + brokerController3.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + + // init consumer offset + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + String messageId = createUniqID(); + SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendMessageRequest(topic, messageId)); + assertSendMessage(sendResponse, messageId); + + this.sendClientSettings(stub, buildSimpleConsumerClientSettings(group)).get(); + + AtomicInteger receiveMessageCount = new AtomicInteger(0); + + assertAndGetReceiveMessage(receiveMessage(blockingStub, topic, group), messageId); + receiveMessageCount.incrementAndGet(); + + DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(group); + defaultMQPullConsumer.start(); + org.apache.rocketmq.common.message.MessageQueue dlqMQ = new org.apache.rocketmq.common.message.MessageQueue(MixAll.getDLQTopic(group), broker1Name, 0); + await().atMost(java.time.Duration.ofSeconds(30)).until(() -> { + try { + List messageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group, 1)); + receiveMessageCount.addAndGet(messageList.size()); + + PullResult pullResult = defaultMQPullConsumer.pull(dlqMQ, "*", 0L, 1); + if (!PullStatus.FOUND.equals(pullResult.getPullStatus())) { + return false; + } + MessageExt messageExt = pullResult.getMsgFoundList().get(0); + return messageId.equals(messageExt.getMsgId()); + } catch (Throwable ignore) { + return false; + } + }); + + assertThat(receiveMessageCount.get()).isEqualTo(maxDeliveryAttempts); + } + + public void testConsumeOrderly() throws Exception { + String topic = initTopicOnSampleTopicBroker(broker1Name, TopicMessageType.FIFO); + String group = MQRandomUtils.getRandomConsumerGroup(); + + SubscriptionGroupConfig groupConfig = brokerController1.getSubscriptionGroupManager().findSubscriptionGroupConfig(group); + groupConfig.setConsumeMessageOrderly(true); + brokerController1.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + brokerController2.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + brokerController3.getSubscriptionGroupManager().updateSubscriptionGroupConfig(groupConfig); + + this.sendClientSettings(stub, buildPushConsumerClientSettings(group)).get(); + receiveMessage(blockingStub, topic, group, 1); + + String messageGroup = "group"; + this.sendClientSettings(stub, buildProducerClientSettings(topic)).get(); + List messageIdList = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String messageId = createUniqID(); + messageIdList.add(messageId); + SendMessageResponse sendResponse = blockingStub.sendMessage(buildSendOrderMessageRequest(topic, messageId, messageGroup)); + assertSendMessage(sendResponse, messageId); + } + + List messageRecvList = new ArrayList<>(); + this.sendClientSettings(stub, buildPushConsumerClientSettings(group)).get(); + await().atMost(java.time.Duration.ofSeconds(20)).until(() -> { + List retryMessageList = getMessageFromReceiveMessageResponse(receiveMessage(blockingStub, topic, group)); + if (retryMessageList.isEmpty()) { + return false; + } + for (Message message : retryMessageList) { + String messageId = message.getSystemProperties().getMessageId(); + messageRecvList.add(messageId); + blockingStub.ackMessage(buildAckMessageRequest(topic, group, + AckMessageEntry.newBuilder().setMessageId(messageId).setReceiptHandle(message.getSystemProperties().getReceiptHandle()).build())); + } + return messageRecvList.size() == messageIdList.size(); + }); + + for (int i = 0; i < messageIdList.size(); i++) { + assertThat(messageRecvList.get(i)).isEqualTo(messageIdList.get(i)); + } + } + + public List receiveMessage(MessagingServiceGrpc.MessagingServiceBlockingStub stub, + String topic, String group) { + return receiveMessage(stub, topic, group, 15); + } + + public List receiveMessage(MessagingServiceGrpc.MessagingServiceBlockingStub stub, + String topic, String group, int timeSeconds) { + List responseList = new ArrayList<>(); + Iterator responseIterator = stub.withDeadlineAfter(timeSeconds, TimeUnit.SECONDS) + .receiveMessage(buildReceiveMessageRequest(topic, group)); + while (responseIterator.hasNext()) { + responseList.add(responseIterator.next()); + } + return responseList; + } + + public List getMessageFromReceiveMessageResponse(List responseList) { + List messageList = new ArrayList<>(); + for (ReceiveMessageResponse response : responseList) { + if (response.hasMessage()) { + messageList.add(response.getMessage()); + } + } + return messageList; + } + + public QueryRouteRequest buildQueryRouteRequest(String topic) { + return QueryRouteRequest.newBuilder() + .setEndpoints(buildEndpoints(port)) + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .build(); + } + + public QueryAssignmentRequest buildQueryAssignmentRequest(String topic, String group) { + return QueryAssignmentRequest.newBuilder() + .setEndpoints(buildEndpoints(port)) + .setTopic(Resource.newBuilder().setName(topic).build()) + .setGroup(Resource.newBuilder().setName(group).build()) + .build(); + } + + public SendMessageRequest buildSendMessageRequest(String topic, String messageId) { + return SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build()) + .build(); + } + + public SendMessageRequest buildSendOrderMessageRequest(String topic, String messageId, String messageGroup) { + return SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.FIFO) + .setMessageGroup(messageGroup) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build()) + .build(); + } + + public SendMessageRequest buildSendBigMessageRequest(String topic, String messageId, int messageSize) { + return SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.NORMAL) + .setBodyEncoding(Encoding.GZIP) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8(RandomUtils.getStringWithCharacter(messageSize))) + .build()) + .build(); + } + + public SendMessageRequest buildTransactionSendMessageRequest(String topic, String messageId) { + return SendMessageRequest.newBuilder() + .addMessages(Message.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setSystemProperties(SystemProperties.newBuilder() + .setMessageId(messageId) + .setQueueId(0) + .setMessageType(MessageType.TRANSACTION) + .setOrphanedTransactionRecoveryDuration(Duration.newBuilder().setSeconds(10)) + .setBornTimestamp(Timestamps.fromMillis(System.currentTimeMillis())) + .setBornHost(StringUtils.defaultString(RemotingUtil.getLocalAddress(), "127.0.0.1:1234")) + .build()) + .setBody(ByteString.copyFromUtf8("123")) + .build()) + .build(); + } + + public ReceiveMessageRequest buildReceiveMessageRequest(String topic, String group) { + return ReceiveMessageRequest.newBuilder() + .setGroup(Resource.newBuilder() + .setName(group) + .build()) + .setMessageQueue(MessageQueue.newBuilder() + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setId(-1) + .build()) + .setBatchSize(1) + .setAutoRenew(false) + .setInvisibleDuration(Duration.newBuilder() + .setSeconds(3) + .build()) + .build(); + } + + public AckMessageRequest buildAckMessageRequest(String topic, String group, AckMessageEntry... entry) { + return AckMessageRequest.newBuilder() + .setGroup(Resource.newBuilder() + .setName(group) + .build()) + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .addAllEntries(Arrays.stream(entry).collect(Collectors.toList())) + .build(); + } + + public EndTransactionRequest buildEndTransactionRequest(String topic, String messageId, String transactionId, + TransactionResolution resolution) { + return EndTransactionRequest.newBuilder() + .setMessageId(messageId) + .setTopic(Resource.newBuilder() + .setName(topic) + .build()) + .setTransactionId(transactionId) + .setResolution(resolution) + .setSource(TransactionSource.SOURCE_SERVER_CHECK) + .build(); + } + + public ChangeInvisibleDurationRequest buildChangeInvisibleDurationRequest(String topic, String group, + String receiptHandle, int second) { + return ChangeInvisibleDurationRequest.newBuilder() + .setTopic(Resource.newBuilder().setName(topic).build()) + .setGroup(Resource.newBuilder().setName(group).build()) + .setInvisibleDuration(Durations.fromSeconds(second)) + .setReceiptHandle(receiptHandle) + .build(); + } + + public void assertQueryRoute(QueryRouteResponse response, int messageQueueSize) { + assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); + assertThat(response.getMessageQueuesList().size()).isEqualTo(messageQueueSize); + assertThat(response.getMessageQueues(0).getBroker().getEndpoints().getAddresses(0).getPort()).isEqualTo(ConfigurationManager.getProxyConfig().getGrpcServerPort()); + } + + public void assertQueryAssignment(QueryAssignmentResponse response, int assignmentCount) { + assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); + assertThat(response.getAssignmentsCount()).isEqualTo(assignmentCount); + assertThat(response.getAssignments(0).getMessageQueue().getBroker().getEndpoints().getAddresses(0).getPort()).isEqualTo(ConfigurationManager.getProxyConfig().getGrpcServerPort()); + } + + public void assertSendMessage(SendMessageResponse response, String messageId) { + assertThat(response.getStatus()).isEqualTo(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); + assertThat(response.getEntries(0).getMessageId()).isEqualTo(messageId); + } + + public Message assertAndGetReceiveMessage(List response, String messageId) { + assertThat(response.get(0).hasStatus()).isTrue(); + assertThat(response.get(0).getStatus() + .getCode()).isEqualTo(Code.OK); + assertThat(response.get(1).getMessage() + .getSystemProperties() + .getMessageId()).isEqualTo(messageId); + return response.get(1).getMessage(); + } + + public void assertRecoverOrphanedTransactionCommand(RecoverOrphanedTransactionCommand command, String messageId) { + assertThat(command.getTransactionId()).isNotBlank(); + } + + public void assertEndTransactionResponse(EndTransactionResponse response) { + assertThat(response.getStatus().getCode()).isEqualTo(Code.OK); + } + + public void assertChangeInvisibleDurationResponse(ChangeInvisibleDurationResponse response, String prevHandle) { + assertThat(response.getStatus().getCode()).isEqualTo(Code.OK); + assertThat(response.getReceiptHandle()).isNotEqualTo(prevHandle); + } + + public Endpoints buildEndpoints(int port) { + return Endpoints.newBuilder() + .setScheme(AddressScheme.IPv4) + .addAddresses(Address.newBuilder() + .setHost("127.0.0.1") + .setPort(port) + .build()) + .build(); + } + + public Settings buildSimpleConsumerClientSettings(String group) { + return Settings.newBuilder() + .setClientType(ClientType.SIMPLE_CONSUMER) + .setSubscription(Subscription.newBuilder() + .setGroup(Resource.newBuilder().setName(group).build()) + .build()) + .build(); + } + + public Settings buildPushConsumerClientSettings(String group) { + return buildPushConsumerClientSettings(2, group); + } + + public Settings buildPushConsumerClientSettings(int maxDeliveryAttempts, String group) { + return Settings.newBuilder() + .setClientType(ClientType.PUSH_CONSUMER) + .setBackoffPolicy(RetryPolicy.newBuilder() + .setMaxAttempts(maxDeliveryAttempts) + .build()) + .setSubscription(Subscription.newBuilder() + .setGroup(Resource.newBuilder().setName(group).build()) + .build()) + .build(); + } + + public Settings buildProducerClientSettings(String... topics) { + List topicResources = Arrays.stream(topics).map(topic -> Resource.newBuilder().setName(topic).build()) + .collect(Collectors.toList()); + return Settings.newBuilder() + .setClientType(ClientType.PRODUCER) + .setPublishing(Publishing.newBuilder() + .addAllTopics(topicResources) + .build()) + .build(); + } + + protected static class DefaultTelemetryCommandStreamObserver implements StreamObserver { + + @Override + public void onNext(TelemetryCommand value) { + + } + + @Override + public void onError(Throwable t) { + + } + + @Override + public void onCompleted() { + + } + } +} \ No newline at end of file diff --git a/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java new file mode 100644 index 0000000000..de72069014 --- /dev/null +++ b/test/src/test/java/org/apache/rocketmq/test/grpc/v2/LocalGrpcIT.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.rocketmq.test.grpc.v2; + +import apache.rocketmq.v2.QueryAssignmentResponse; +import apache.rocketmq.v2.QueryRouteResponse; +import org.apache.rocketmq.proxy.config.ConfigurationManager; +import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication; +import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor; +import org.apache.rocketmq.proxy.processor.MessagingProcessor; +import org.junit.After; +import org.junit.Before; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(value = MethodSorters.NAME_ASCENDING) +public class LocalGrpcIT extends GrpcBaseIT { + + private MessagingProcessor messagingProcessor; + private GrpcMessagingApplication grpcMessagingApplication; + + @Before + public void setUp() throws Exception { + super.setUp(); + messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController1); + messagingProcessor.start(); + grpcMessagingApplication = GrpcMessagingApplication.create(messagingProcessor); + grpcMessagingApplication.start(); + setUpServer(grpcMessagingApplication, ConfigurationManager.getProxyConfig().getGrpcServerPort(), true); + } + + @After + public void clean() throws Exception { + messagingProcessor.shutdown(); + grpcMessagingApplication.shutdown(); + shutdown(); + } + + @Test + public void testQueryRoute() throws Exception { + String topic = initTopic(); + + QueryRouteResponse response = blockingStub.queryRoute(buildQueryRouteRequest(topic)); + assertQueryRoute(response, brokerControllerList.size() * DEFAULT_QUEUE_NUMS); + } + + @Test + public void testQueryAssignment() throws Exception { + String topic = initTopic(); + String group = "group"; + + QueryAssignmentResponse response = blockingStub.queryAssignment(buildQueryAssignmentRequest(topic, group)); + + assertQueryAssignment(response, brokerNum); + } + + @Test + public void testTransactionCheckThenCommit() { + super.testTransactionCheckThenCommit(); + } + + @Test + public void testSimpleConsumerSendAndRecvDelayMessage() throws Exception { + super.testSimpleConsumerSendAndRecvDelayMessage(); + } + + @Test + public void testSimpleConsumerSendAndRecvBigMessage() throws Exception { + super.testSimpleConsumerSendAndRecvBigMessage(); + } + + @Test + public void testSimpleConsumerSendAndRecv() throws Exception { + super.testSimpleConsumerSendAndRecv(); + } + + @Test + public void testSimpleConsumerToDLQ() throws Exception { + super.testSimpleConsumerToDLQ(); + } + + @Test + public void testConsumeOrderly() throws Exception { + super.testConsumeOrderly(); + } +} diff --git a/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java index 29b92d5f4a..be0aebec83 100644 --- a/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java +++ b/test/src/test/java/org/apache/rocketmq/test/statictopic/StaticTopicIT.java @@ -345,7 +345,7 @@ public class StaticTopicIT extends BaseConf { RMQNormalConsumer consumer = getConsumer(nsAddr, group, topic, "*", new RMQNormalListener()); long start = System.currentTimeMillis(); - int queueNum = 10; + int queueNum = 5; int msgEachQueue = 10; //create static topic { diff --git a/test/src/test/resources/rmq-proxy-home/conf/broker.conf b/test/src/test/resources/rmq-proxy-home/conf/broker.conf new file mode 100644 index 0000000000..0c0b28b7b8 --- /dev/null +++ b/test/src/test/resources/rmq-proxy-home/conf/broker.conf @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +brokerClusterName = DefaultCluster +brokerName = broker-a +brokerId = 0 +deleteWhen = 04 +fileReservedTime = 48 +brokerRole = ASYNC_MASTER +flushDiskType = ASYNC_FLUSH diff --git a/test/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml b/test/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml new file mode 100644 index 0000000000..8d0458ebf0 --- /dev/null +++ b/test/src/test/resources/rmq-proxy-home/conf/logback_proxy.xml @@ -0,0 +1,420 @@ + + + + + + + ${user.home}/logs/rocketmqlogs/proxy.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/proxy.%i.log.gz + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/grpc.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/grpc.%i.log.gz + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker_default.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker.%i.log.gz + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/protection.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/protection.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/watermark.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/watermark.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/store.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/store.%i.log.gz + 1 + 10 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/remoting.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/remoting.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/storeerror.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/storeerror.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/transaction.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/transaction.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/lock.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/lock.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/filter.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/filter.%i.log.gz + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/stats.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/stats.%i.log.gz + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/${brokerLogDir}/commercial.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/commercial.%i.log.gz + 1 + 10 + + + 500MB + + + + + ${user.home}/logs/rocketmqlogs/pop.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/pop.%i.log + + 1 + 20 + + + 128MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json b/test/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json new file mode 100644 index 0000000000..f0873e2a35 --- /dev/null +++ b/test/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json @@ -0,0 +1,3 @@ +{ + "proxyMode": "cluster" +} \ No newline at end of file