Merge pull request #4225 from drpmma/feature/grpc-develop-v2

[ISSUE #3949] Support gRPC protocol

.licenserc.yaml

@@ -34,12 +34,14 @@ header:
     - 'src/test/**/*.log'
     - '*/src/test/resources/META-INF/service/*'
     - '*/src/main/resources/META-INF/service/*'
+    - '*/src/test/resources/rmq-proxy-home/conf/rmq-proxy.json'
     - '**/target/**'
     - '**/*.iml'
     - 'docs/**'
     - 'localbin/**'
     - 'distribution/LICENSE-BIN'
     - 'distribution/NOTICE-BIN'
+    - 'distribution/conf/rmq-proxy.json'
 
 
   comment: on-failure

acl/pom.xml

@@ -19,6 +19,10 @@
     <name>rocketmq-acl ${project.version}</name>
 
     <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>rocketmq-proto</artifactId>
+        </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>rocketmq-remoting</artifactId>
@@ -62,6 +66,10 @@
             <groupId>commons-validator</groupId>
             <artifactId>commons-validator</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java-util</artifactId>
+        </dependency>
 
     </dependencies>
 </project>

acl/src/main/java/org/apache/rocketmq/acl/AccessValidator.java

@@ -17,9 +17,10 @@
 
 package org.apache.rocketmq.acl;
 
+import com.google.protobuf.GeneratedMessageV3;
 import java.util.List;
 import java.util.Map;
-
+import org.apache.rocketmq.acl.common.AuthenticationHeader;
 import org.apache.rocketmq.common.AclConfig;
 import org.apache.rocketmq.common.DataVersion;
 import org.apache.rocketmq.common.PlainAccessConfig;
@@ -36,6 +37,14 @@ public interface AccessValidator {
      */
     AccessResource parse(RemotingCommand request, String remoteAddr);
 
+    /**
+     * Parse to get the AccessResource from gRPC protocol
+     * @param messageV3
+     * @param header
+     * @return Plain access resource
+     */
+    AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header);
+
     /**
      * Validate the access resource.
      *

acl/src/main/java/org/apache/rocketmq/acl/PermissionChecker.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.acl;
+
+public interface PermissionChecker {
+    void check(AccessResource checkedAccess, AccessResource ownedAccess);
+}

acl/src/main/java/org/apache/rocketmq/acl/common/AuthenticationHeader.java

@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.acl.common;
+
+import com.google.common.base.MoreObjects;
+
+public class AuthenticationHeader {
+    private String remoteAddress;
+    private String tenantId;
+    private String namespace;
+    private String authorization;
+    private String datetime;
+    private String sessionToken;
+    private String requestId;
+    private String language;
+    private String clientVersion;
+    private String protocol;
+    private int requestCode;
+
+    AuthenticationHeader(final String remoteAddress, final String tenantId, final String namespace,
+        final String authorization, final String datetime, final String sessionToken, final String requestId,
+        final String language, final String clientVersion, final String protocol, final int requestCode) {
+        this.remoteAddress = remoteAddress;
+        this.tenantId = tenantId;
+        this.namespace = namespace;
+        this.authorization = authorization;
+        this.datetime = datetime;
+        this.sessionToken = sessionToken;
+        this.requestId = requestId;
+        this.language = language;
+        this.clientVersion = clientVersion;
+        this.protocol = protocol;
+        this.requestCode = requestCode;
+    }
+
+    public static class MetadataHeaderBuilder {
+        private String remoteAddress;
+        private String tenantId;
+        private String namespace;
+        private String authorization;
+        private String datetime;
+        private String sessionToken;
+        private String requestId;
+        private String language;
+        private String clientVersion;
+        private String protocol;
+        private int requestCode;
+
+        MetadataHeaderBuilder() {
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder remoteAddress(final String remoteAddress) {
+            this.remoteAddress = remoteAddress;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder tenantId(final String tenantId) {
+            this.tenantId = tenantId;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder namespace(final String namespace) {
+            this.namespace = namespace;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder authorization(final String authorization) {
+            this.authorization = authorization;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder datetime(final String datetime) {
+            this.datetime = datetime;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder sessionToken(final String sessionToken) {
+            this.sessionToken = sessionToken;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder requestId(final String requestId) {
+            this.requestId = requestId;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder language(final String language) {
+            this.language = language;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder clientVersion(final String clientVersion) {
+            this.clientVersion = clientVersion;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder protocol(final String protocol) {
+            this.protocol = protocol;
+            return this;
+        }
+
+        public AuthenticationHeader.MetadataHeaderBuilder requestCode(final int requestCode) {
+            this.requestCode = requestCode;
+            return this;
+        }
+
+        public AuthenticationHeader build() {
+            return new AuthenticationHeader(this.remoteAddress, this.tenantId, this.namespace, this.authorization,
+                this.datetime, this.sessionToken, this.requestId, this.language, this.clientVersion, this.protocol,
+                this.requestCode);
+        }
+    }
+
+    public static AuthenticationHeader.MetadataHeaderBuilder builder() {
+        return new AuthenticationHeader.MetadataHeaderBuilder();
+    }
+
+    public String getRemoteAddress() {
+        return this.remoteAddress;
+    }
+
+    public String getTenantId() {
+        return this.tenantId;
+    }
+
+    public String getNamespace() {
+        return this.namespace;
+    }
+
+    public String getAuthorization() {
+        return this.authorization;
+    }
+
+    public String getDatetime() {
+        return this.datetime;
+    }
+
+    public String getSessionToken() {
+        return this.sessionToken;
+    }
+
+    public String getRequestId() {
+        return this.requestId;
+    }
+
+    public String getLanguage() {
+        return this.language;
+    }
+
+    public String getClientVersion() {
+        return this.clientVersion;
+    }
+
+    public String getProtocol() {
+        return this.protocol;
+    }
+
+    public int getRequestCode() {
+        return this.requestCode;
+    }
+
+    public void setRemoteAddress(final String remoteAddress) {
+        this.remoteAddress = remoteAddress;
+    }
+
+    public void setTenantId(final String tenantId) {
+        this.tenantId = tenantId;
+    }
+
+    public void setNamespace(final String namespace) {
+        this.namespace = namespace;
+    }
+
+    public void setAuthorization(final String authorization) {
+        this.authorization = authorization;
+    }
+
+    public void setDatetime(final String datetime) {
+        this.datetime = datetime;
+    }
+
+    public void setSessionToken(final String sessionToken) {
+        this.sessionToken = sessionToken;
+    }
+
+    public void setRequestId(final String requestId) {
+        this.requestId = requestId;
+    }
+
+    public void setLanguage(final String language) {
+        this.language = language;
+    }
+
+    public void setClientVersion(final String clientVersion) {
+        this.clientVersion = clientVersion;
+    }
+
+    public void setProtocol(final String protocol) {
+        this.protocol = protocol;
+    }
+
+    public void setRequestCode(int requestCode) {
+        this.requestCode = requestCode;
+    }
+
+    @Override public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("remoteAddress", remoteAddress)
+            .add("tenantId", tenantId)
+            .add("namespace", namespace)
+            .add("authorization", authorization)
+            .add("datetime", datetime)
+            .add("sessionToken", sessionToken)
+            .add("requestId", requestId)
+            .add("language", language)
+            .add("clientVersion", clientVersion)
+            .add("protocol", protocol)
+            .add("requestCode", requestCode)
+            .toString();
+    }
+}

acl/src/main/java/org/apache/rocketmq/acl/common/AuthorizationHeader.java

@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.acl.common;
+
+import com.google.common.base.MoreObjects;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+
+public class AuthorizationHeader {
+    private static final String HEADER_SEPARATOR = " ";
+    private static final String CREDENTIALS_SEPARATOR = "/";
+    private static final int AUTH_HEADER_KV_LENGTH = 2;
+    private static final String CREDENTIAL = "Credential";
+    private static final String SIGNED_HEADERS = "SignedHeaders";
+    private static final String SIGNATURE = "Signature";
+    private String method;
+    private String accessKey;
+    private String[] signedHeaders;
+    private String signature;
+
+    /**
+     * Parse authorization from gRPC header.
+     *
+     * @param header gRPC header string.
+     * @throws Exception exception.
+     */
+    public AuthorizationHeader(String header) throws DecoderException {
+        String[] result = header.split(HEADER_SEPARATOR, 2);
+        if (result.length != 2) {
+            throw new DecoderException("authorization header is incorrect");
+        }
+        this.method = result[0];
+        String[] keyValues = result[1].split(",");
+        for (String keyValue : keyValues) {
+            String[] kv = keyValue.trim().split("=", 2);
+            int kvLength = kv.length;
+            if (kv.length != AUTH_HEADER_KV_LENGTH) {
+                throw new DecoderException("authorization keyValues length is incorrect, actual length=" + kvLength);
+            }
+            String authItem = kv[0];
+            if (CREDENTIAL.equals(authItem)) {
+                String[] credential = kv[1].split(CREDENTIALS_SEPARATOR);
+                int credentialActualLength = credential.length;
+                if (credentialActualLength == 0) {
+                    throw new DecoderException("authorization credential length is incorrect, actual length=" + credentialActualLength);
+                }
+                this.accessKey = credential[0];
+                continue;
+            }
+            if (SIGNED_HEADERS.equals(authItem)) {
+                this.signedHeaders = kv[1].split(";");
+                continue;
+            }
+            if (SIGNATURE.equals(authItem)) {
+                this.signature = this.hexToBase64(kv[1]);
+            }
+        }
+    }
+
+    public String hexToBase64(String input) throws DecoderException {
+        byte[] bytes = Hex.decodeHex(input);
+        return Base64.encodeBase64String(bytes);
+    }
+
+    public String getMethod() {
+        return this.method;
+    }
+
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public String[] getSignedHeaders() {
+        return this.signedHeaders;
+    }
+
+    public String getSignature() {
+        return this.signature;
+    }
+
+    public void setMethod(final String method) {
+        this.method = method;
+    }
+
+    public void setAccessKey(final String accessKey) {
+        this.accessKey = accessKey;
+    }
+
+    public void setSignedHeaders(final String[] signedHeaders) {
+        this.signedHeaders = signedHeaders;
+    }
+
+    public void setSignature(final String signature) {
+        this.signature = signature;
+    }
+
+    @Override public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("method", method)
+            .add("accessKey", accessKey)
+            .add("signedHeaders", signedHeaders)
+            .add("signature", signature)
+            .toString();
+    }
+}

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessResource.java

@@ -16,11 +16,46 @@
  */
 package org.apache.rocketmq.acl.plain;
 
+import apache.rocketmq.v2.AckMessageRequest;
+import apache.rocketmq.v2.EndTransactionRequest;
+import apache.rocketmq.v2.ForwardMessageToDeadLetterQueueRequest;
+import apache.rocketmq.v2.HeartbeatRequest;
+import apache.rocketmq.v2.Message;
+import apache.rocketmq.v2.ReceiveMessageRequest;
+import apache.rocketmq.v2.Resource;
+import apache.rocketmq.v2.SendMessageRequest;
+import apache.rocketmq.v2.Subscription;
+import apache.rocketmq.v2.SubscriptionEntry;
+import apache.rocketmq.v2.TelemetryCommand;
+import com.google.protobuf.GeneratedMessageV3;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import org.apache.commons.codec.DecoderException;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.rocketmq.acl.AccessResource;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.AclUtils;
+import org.apache.rocketmq.acl.common.AuthenticationHeader;
+import org.apache.rocketmq.acl.common.AuthorizationHeader;
+import org.apache.rocketmq.acl.common.Permission;
+import org.apache.rocketmq.acl.common.SessionCredentials;
 import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.common.protocol.NamespaceUtil;
+import org.apache.rocketmq.common.protocol.RequestCode;
+import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
+import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
+import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
+import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+import org.apache.rocketmq.remoting.common.RemotingHelper;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public class PlainAccessResource implements AccessResource {
 
@@ -55,6 +90,201 @@ public class PlainAccessResource implements AccessResource {
     public PlainAccessResource() {
     }
 
+    public static PlainAccessResource parse(RemotingCommand request, String remoteAddr) {
+        PlainAccessResource accessResource = new PlainAccessResource();
+        if (remoteAddr != null && remoteAddr.contains(":")) {
+            accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':')));
+        } else {
+            accessResource.setWhiteRemoteAddress(remoteAddr);
+        }
+
+        accessResource.setRequestCode(request.getCode());
+
+        if (request.getExtFields() == null) {
+            // If request's extFields is null,then return accessResource directly(users can use whiteAddress pattern)
+            // The following logic codes depend on the request's extFields not to be null.
+            return accessResource;
+        }
+        accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY));
+        accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE));
+        accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN));
+
+        try {
+            switch (request.getCode()) {
+                case RequestCode.SEND_MESSAGE:
+                    final String topic = request.getExtFields().get("topic");
+                    if (PlainAccessResource.isRetryTopic(topic)) {
+                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
+                    } else {
+                        accessResource.addResourceAndPerm(topic, Permission.PUB);
+                    }
+                    break;
+                case RequestCode.SEND_MESSAGE_V2:
+                    final String topicV2 = request.getExtFields().get("b");
+                    if (PlainAccessResource.isRetryTopic(topicV2)) {
+                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("a")), Permission.SUB);
+                    } else {
+                        accessResource.addResourceAndPerm(topicV2, Permission.PUB);
+                    }
+                    break;
+                case RequestCode.CONSUMER_SEND_MSG_BACK:
+                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
+                    break;
+                case RequestCode.PULL_MESSAGE:
+                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
+                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB);
+                    break;
+                case RequestCode.QUERY_MESSAGE:
+                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
+                    break;
+                case RequestCode.HEART_BEAT:
+                    HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);
+                    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
+                        accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB);
+                        for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) {
+                            accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB);
+                        }
+                    }
+                    break;
+                case RequestCode.UNREGISTER_CLIENT:
+                    final UnregisterClientRequestHeader unregisterClientRequestHeader =
+                        (UnregisterClientRequestHeader) request
+                            .decodeCommandCustomHeader(UnregisterClientRequestHeader.class);
+                    accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB);
+                    break;
+                case RequestCode.GET_CONSUMER_LIST_BY_GROUP:
+                    final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader =
+                        (GetConsumerListByGroupRequestHeader) request
+                            .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);
+                    accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB);
+                    break;
+                case RequestCode.UPDATE_CONSUMER_OFFSET:
+                    final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader =
+                        (UpdateConsumerOffsetRequestHeader) request
+                            .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
+                    accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB);
+                    accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB);
+                    break;
+                default:
+                    break;
+
+            }
+        } catch (Throwable t) {
+            throw new AclException(t.getMessage(), t);
+        }
+
+        // Content
+        SortedMap<String, String> map = new TreeMap<String, String>();
+        for (Map.Entry<String, String> entry : request.getExtFields().entrySet()) {
+            if (!SessionCredentials.SIGNATURE.equals(entry.getKey())
+                && !MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) {
+                map.put(entry.getKey(), entry.getValue());
+            }
+        }
+        accessResource.setContent(AclUtils.combineRequestContent(request, map));
+        return accessResource;
+    }
+
+    public static PlainAccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) {
+        PlainAccessResource accessResource = new PlainAccessResource();
+        String remoteAddress = header.getRemoteAddress();
+        if (remoteAddress != null && remoteAddress.contains(":")) {
+            accessResource.setWhiteRemoteAddress(RemotingHelper.parseHostFromAddress(remoteAddress));
+        } else {
+            accessResource.setWhiteRemoteAddress(remoteAddress);
+        }
+        try {
+            AuthorizationHeader authorizationHeader = new AuthorizationHeader(header.getAuthorization());
+            accessResource.setAccessKey(authorizationHeader.getAccessKey());
+            accessResource.setSignature(authorizationHeader.getSignature());
+        } catch (DecoderException e) {
+            throw new AclException(e.getMessage(), e);
+        }
+        accessResource.setSecretToken(header.getSessionToken());
+        accessResource.setRequestCode(header.getRequestCode());
+        accessResource.setContent(header.getDatetime().getBytes(StandardCharsets.UTF_8));
+
+        try {
+            String rpcFullName = messageV3.getDescriptorForType().getFullName();
+            if (HeartbeatRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                HeartbeatRequest request = (HeartbeatRequest) messageV3;
+                if (request.hasGroup()) {
+                    accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB);
+                }
+            } else if (SendMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                SendMessageRequest request = (SendMessageRequest) messageV3;
+                if (request.getMessagesCount() <= 0) {
+                    throw new AclException("SendMessageRequest, messageCount is zero", ResponseCode.MESSAGE_ILLEGAL);
+                }
+                Resource topic = request.getMessages(0).getTopic();
+                for (Message message : request.getMessagesList()) {
+                    if (!message.getTopic().equals(topic)) {
+                        throw new AclException("SendMessageRequest, messages' topic is not consistent", ResponseCode.MESSAGE_ILLEGAL);
+                    }
+                }
+                accessResource.addResourceAndPerm(topic, Permission.PUB);
+            } else if (ReceiveMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                ReceiveMessageRequest request = (ReceiveMessageRequest) messageV3;
+                accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB);
+                accessResource.addResourceAndPerm(request.getMessageQueue().getTopic(), Permission.SUB);
+            } else if (AckMessageRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                AckMessageRequest request = (AckMessageRequest) messageV3;
+                accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB);
+                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
+            } else if (ForwardMessageToDeadLetterQueueRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                ForwardMessageToDeadLetterQueueRequest request = (ForwardMessageToDeadLetterQueueRequest) messageV3;
+                accessResource.addResourceAndPerm(request.getGroup(), Permission.SUB);
+                accessResource.addResourceAndPerm(request.getTopic(), Permission.SUB);
+            } else if (EndTransactionRequest.getDescriptor().getFullName().equals(rpcFullName)) {
+                EndTransactionRequest request = (EndTransactionRequest) messageV3;
+                accessResource.addResourceAndPerm(request.getTopic(), Permission.PUB);
+            } else if (TelemetryCommand.getDescriptor().getFullName().equals(rpcFullName)) {
+                TelemetryCommand command = (TelemetryCommand) messageV3;
+                if (command.getCommandCase() == TelemetryCommand.CommandCase.SETTINGS) {
+                    if (command.getSettings().hasPublishing()) {
+                        List<Resource> topicList = command.getSettings().getPublishing().getTopicsList();
+                        for (Resource topic : topicList) {
+                            accessResource.addResourceAndPerm(topic, Permission.PUB);
+                        }
+                    }
+                    if (command.getSettings().hasSubscription()) {
+                        Subscription subscription = command.getSettings().getSubscription();
+                        accessResource.addResourceAndPerm(subscription.getGroup(), Permission.SUB);
+                        for (SubscriptionEntry entry : subscription.getSubscriptionsList()) {
+                            accessResource.addResourceAndPerm(entry.getTopic(), Permission.SUB);
+                        }
+                    }
+                }
+            }
+        } catch (Throwable t) {
+            throw new AclException(t.getMessage(), t);
+        }
+        return accessResource;
+    }
+
+    private void addResourceAndPerm(Resource resource, byte permission) {
+        String resourceName = NamespaceUtil.wrapNamespace(resource.getResourceNamespace(), resource.getName());
+        addResourceAndPerm(resourceName, permission);
+    }
+
+    public static PlainAccessResource build(PlainAccessConfig plainAccessConfig, RemoteAddressStrategy remoteAddressStrategy) {
+        PlainAccessResource plainAccessResource = new PlainAccessResource();
+        plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey());
+        plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey());
+        plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
+
+        plainAccessResource.setAdmin(plainAccessConfig.isAdmin());
+
+        plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm()));
+        plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm()));
+
+        Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms());
+        Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms());
+
+        plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategy);
+        return plainAccessResource;
+    }
+
     public static boolean isRetryTopic(String topic) {
         return null != topic && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX);
     }

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainAccessValidator.java

@@ -16,31 +16,16 @@
  */
 package org.apache.rocketmq.acl.plain;
 
-import org.apache.rocketmq.acl.AccessResource;
-import org.apache.rocketmq.acl.AccessValidator;
-import org.apache.rocketmq.acl.common.AclException;
-import org.apache.rocketmq.acl.common.AclUtils;
-import org.apache.rocketmq.acl.common.Permission;
-import org.apache.rocketmq.acl.common.SessionCredentials;
-import org.apache.rocketmq.common.AclConfig;
-import org.apache.rocketmq.common.DataVersion;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.PlainAccessConfig;
-import org.apache.rocketmq.common.protocol.RequestCode;
-import org.apache.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader;
-import org.apache.rocketmq.common.protocol.header.UnregisterClientRequestHeader;
-import org.apache.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader;
-import org.apache.rocketmq.common.protocol.heartbeat.ConsumerData;
-import org.apache.rocketmq.common.protocol.heartbeat.HeartbeatData;
-import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
-import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-
+import com.google.protobuf.GeneratedMessageV3;
 import java.util.List;
 import java.util.Map;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import static org.apache.rocketmq.acl.plain.PlainAccessResource.getRetryTopic;
+import org.apache.rocketmq.acl.AccessResource;
+import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.acl.common.AuthenticationHeader;
+import org.apache.rocketmq.common.AclConfig;
+import org.apache.rocketmq.common.DataVersion;
+import org.apache.rocketmq.common.PlainAccessConfig;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public class PlainAccessValidator implements AccessValidator {
 
@@ -52,98 +37,12 @@ public class PlainAccessValidator implements AccessValidator {
 
     @Override
     public AccessResource parse(RemotingCommand request, String remoteAddr) {
-        PlainAccessResource accessResource = new PlainAccessResource();
-        if (remoteAddr != null && remoteAddr.contains(":")) {
-            accessResource.setWhiteRemoteAddress(remoteAddr.substring(0, remoteAddr.lastIndexOf(':')));
-        } else {
-            accessResource.setWhiteRemoteAddress(remoteAddr);
-        }
+        return PlainAccessResource.parse(request, remoteAddr);
+    }
 
-        accessResource.setRequestCode(request.getCode());
-
-        if (request.getExtFields() == null) {
-            // If request's extFields is null,then return accessResource directly(users can use whiteAddress pattern)
-            // The following logic codes depend on the request's extFields not to be null.
-            return accessResource;
-        }
-        accessResource.setAccessKey(request.getExtFields().get(SessionCredentials.ACCESS_KEY));
-        accessResource.setSignature(request.getExtFields().get(SessionCredentials.SIGNATURE));
-        accessResource.setSecretToken(request.getExtFields().get(SessionCredentials.SECURITY_TOKEN));
-
-        try {
-            switch (request.getCode()) {
-                case RequestCode.SEND_MESSAGE:
-                    final String topic = request.getExtFields().get("topic");
-                    if (PlainAccessResource.isRetryTopic(topic)) {
-                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
-                    } else {
-                        accessResource.addResourceAndPerm(topic, Permission.PUB);
-                    }
-                    break;
-                case RequestCode.SEND_MESSAGE_V2:
-                    final String topicV2 = request.getExtFields().get("b");
-                    if (PlainAccessResource.isRetryTopic(topicV2)) {
-                        accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("a")), Permission.SUB);
-                    } else {
-                        accessResource.addResourceAndPerm(topicV2, Permission.PUB);
-                    }
-                    break;
-                case RequestCode.CONSUMER_SEND_MSG_BACK:
-                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("group")), Permission.SUB);
-                    break;
-                case RequestCode.PULL_MESSAGE:
-                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
-                    accessResource.addResourceAndPerm(getRetryTopic(request.getExtFields().get("consumerGroup")), Permission.SUB);
-                    break;
-                case RequestCode.QUERY_MESSAGE:
-                    accessResource.addResourceAndPerm(request.getExtFields().get("topic"), Permission.SUB);
-                    break;
-                case RequestCode.HEART_BEAT:
-                    HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class);
-                    for (ConsumerData data : heartbeatData.getConsumerDataSet()) {
-                        accessResource.addResourceAndPerm(getRetryTopic(data.getGroupName()), Permission.SUB);
-                        for (SubscriptionData subscriptionData : data.getSubscriptionDataSet()) {
-                            accessResource.addResourceAndPerm(subscriptionData.getTopic(), Permission.SUB);
-                        }
-                    }
-                    break;
-                case RequestCode.UNREGISTER_CLIENT:
-                    final UnregisterClientRequestHeader unregisterClientRequestHeader =
-                        (UnregisterClientRequestHeader) request
-                            .decodeCommandCustomHeader(UnregisterClientRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(unregisterClientRequestHeader.getConsumerGroup()), Permission.SUB);
-                    break;
-                case RequestCode.GET_CONSUMER_LIST_BY_GROUP:
-                    final GetConsumerListByGroupRequestHeader getConsumerListByGroupRequestHeader =
-                        (GetConsumerListByGroupRequestHeader) request
-                            .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(getConsumerListByGroupRequestHeader.getConsumerGroup()), Permission.SUB);
-                    break;
-                case RequestCode.UPDATE_CONSUMER_OFFSET:
-                    final UpdateConsumerOffsetRequestHeader updateConsumerOffsetRequestHeader =
-                        (UpdateConsumerOffsetRequestHeader) request
-                            .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class);
-                    accessResource.addResourceAndPerm(getRetryTopic(updateConsumerOffsetRequestHeader.getConsumerGroup()), Permission.SUB);
-                    accessResource.addResourceAndPerm(updateConsumerOffsetRequestHeader.getTopic(), Permission.SUB);
-                    break;
-                default:
-                    break;
-
-            }
-        } catch (Throwable t) {
-            throw new AclException(t.getMessage(), t);
-        }
-
-        // Content
-        SortedMap<String, String> map = new TreeMap<String, String>();
-        for (Map.Entry<String, String> entry : request.getExtFields().entrySet()) {
-            if (!SessionCredentials.SIGNATURE.equals(entry.getKey())
-                && !MixAll.UNIQUE_MSG_QUERY_FLAG.equals(entry.getKey())) {
-                map.put(entry.getKey(), entry.getValue());
-            }
-        }
-        accessResource.setContent(AclUtils.combineRequestContent(request, map));
-        return accessResource;
+    @Override
+    public AccessResource parse(GeneratedMessageV3 messageV3, AuthenticationHeader header) {
+        return PlainAccessResource.parse(messageV3, header);
     }
 
     @Override
@@ -161,22 +60,26 @@ public class PlainAccessValidator implements AccessValidator {
         return aclPlugEngine.deleteAccessConfig(accesskey);
     }
 
-    @Override public String  getAclConfigVersion() {
+    @Override
+    public String getAclConfigVersion() {
         return aclPlugEngine.getAclConfigDataVersion();
     }
 
-    @Override public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList) {
+    @Override
+    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList) {
         return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList);
     }
 
-    @Override public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String aclFileFullPath) {
+    @Override
+    public boolean updateGlobalWhiteAddrsConfig(List<String> globalWhiteAddrsList, String aclFileFullPath) {
         return aclPlugEngine.updateGlobalWhiteAddrsConfig(globalWhiteAddrsList, aclFileFullPath);
     }
 
-    @Override public AclConfig getAllAclConfig() {
+    @Override
+    public AclConfig getAllAclConfig() {
         return aclPlugEngine.getAllAclConfig();
     }
-    
+
     @Override
     public Map<String, DataVersion> getAllAclConfigVersion() {
         return aclPlugEngine.getDataVersionMap();

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionChecker.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.acl.plain;
+
+import java.util.Map;
+import org.apache.rocketmq.acl.AccessResource;
+import org.apache.rocketmq.acl.PermissionChecker;
+import org.apache.rocketmq.acl.common.AclException;
+import org.apache.rocketmq.acl.common.Permission;
+
+public class PlainPermissionChecker implements PermissionChecker {
+    public void check(AccessResource checkedAccess, AccessResource ownedAccess) {
+        PlainAccessResource checkedPlainAccess = (PlainAccessResource) checkedAccess;
+        PlainAccessResource ownedPlainAccess = (PlainAccessResource) ownedAccess;
+        if (Permission.needAdminPerm(checkedPlainAccess.getRequestCode()) && !ownedPlainAccess.isAdmin()) {
+            throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", checkedPlainAccess.getRequestCode(), ownedPlainAccess.getAccessKey()));
+        }
+        Map<String, Byte> needCheckedPermMap = checkedPlainAccess.getResourcePermMap();
+        Map<String, Byte> ownedPermMap = ownedPlainAccess.getResourcePermMap();
+
+        if (needCheckedPermMap == null) {
+            // If the needCheckedPermMap is null,then return
+            return;
+        }
+
+        if (ownedPermMap == null && ownedPlainAccess.isAdmin()) {
+            // If the ownedPermMap is null and it is an admin user, then return
+            return;
+        }
+
+        for (Map.Entry<String, Byte> needCheckedEntry : needCheckedPermMap.entrySet()) {
+            String resource = needCheckedEntry.getKey();
+            Byte neededPerm = needCheckedEntry.getValue();
+            boolean isGroup = PlainAccessResource.isRetryTopic(resource);
+
+            if (ownedPermMap == null || !ownedPermMap.containsKey(resource)) {
+                // Check the default perm
+                byte ownedPerm = isGroup ? ownedPlainAccess.getDefaultGroupPerm() :
+                    ownedPlainAccess.getDefaultTopicPerm();
+                if (!Permission.checkPermission(neededPerm, ownedPerm)) {
+                    throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup)));
+                }
+                continue;
+            }
+            if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) {
+                throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup)));
+            }
+        }
+    }
+}

acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java

@@ -37,6 +37,7 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.acl.PermissionChecker;
 import org.apache.rocketmq.acl.common.AclConstants;
 import org.apache.rocketmq.acl.common.AclException;
 import org.apache.rocketmq.acl.common.AclUtils;
@@ -81,6 +82,8 @@ public class PlainPermissionManager {
 
     private List<String> fileList = new ArrayList<>();
 
+    private final PermissionChecker permissionChecker = new PlainPermissionChecker();
+
     public PlainPermissionManager() {
         this.defaultAclDir = MixAll.dealFilePath(fileHome + File.separator + "conf" + File.separator + "acl");
         this.defaultAclFile = MixAll.dealFilePath(fileHome + File.separator + System.getProperty("rocketmq.acl.plain.file", "conf/plain_acl.yml"));
@@ -576,40 +579,7 @@ public class PlainPermissionManager {
     }
 
     void checkPerm(PlainAccessResource needCheckedAccess, PlainAccessResource ownedAccess) {
-        if (Permission.needAdminPerm(needCheckedAccess.getRequestCode()) && !ownedAccess.isAdmin()) {
-            throw new AclException(String.format("Need admin permission for request code=%d, but accessKey=%s is not", needCheckedAccess.getRequestCode(), ownedAccess.getAccessKey()));
-        }
-        Map<String, Byte> needCheckedPermMap = needCheckedAccess.getResourcePermMap();
-        Map<String, Byte> ownedPermMap = ownedAccess.getResourcePermMap();
-
-        if (needCheckedPermMap == null) {
-            // If the needCheckedPermMap is null,then return
-            return;
-        }
-
-        if (ownedPermMap == null && ownedAccess.isAdmin()) {
-            // If the ownedPermMap is null and it is an admin user, then return
-            return;
-        }
-
-        for (Map.Entry<String, Byte> needCheckedEntry : needCheckedPermMap.entrySet()) {
-            String resource = needCheckedEntry.getKey();
-            Byte neededPerm = needCheckedEntry.getValue();
-            boolean isGroup = PlainAccessResource.isRetryTopic(resource);
-
-            if (ownedPermMap == null || !ownedPermMap.containsKey(resource)) {
-                // Check the default perm
-                byte ownedPerm = isGroup ? ownedAccess.getDefaultGroupPerm() :
-                    ownedAccess.getDefaultTopicPerm();
-                if (!Permission.checkPermission(neededPerm, ownedPerm)) {
-                    throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup)));
-                }
-                continue;
-            }
-            if (!Permission.checkPermission(neededPerm, ownedPermMap.get(resource))) {
-                throw new AclException(String.format("No default permission for %s", PlainAccessResource.printStr(resource, isGroup)));
-            }
-        }
+        permissionChecker.check(needCheckedAccess, ownedAccess);
     }
 
     void clearPermissionInfo() {
@@ -631,23 +601,8 @@ public class PlainPermissionManager {
 
     public PlainAccessResource buildPlainAccessResource(PlainAccessConfig plainAccessConfig) throws AclException {
         checkPlainAccessConfig(plainAccessConfig);
-        PlainAccessResource plainAccessResource = new PlainAccessResource();
-        plainAccessResource.setAccessKey(plainAccessConfig.getAccessKey());
-        plainAccessResource.setSecretKey(plainAccessConfig.getSecretKey());
-        plainAccessResource.setWhiteRemoteAddress(plainAccessConfig.getWhiteRemoteAddress());
-
-        plainAccessResource.setAdmin(plainAccessConfig.isAdmin());
-
-        plainAccessResource.setDefaultGroupPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultGroupPerm()));
-        plainAccessResource.setDefaultTopicPerm(Permission.parsePermFromString(plainAccessConfig.getDefaultTopicPerm()));
-
-        Permission.parseResourcePerms(plainAccessResource, false, plainAccessConfig.getGroupPerms());
-        Permission.parseResourcePerms(plainAccessResource, true, plainAccessConfig.getTopicPerms());
-
-        plainAccessResource.setRemoteAddressStrategy(remoteAddressStrategyFactory.
-            getRemoteAddressStrategy(plainAccessResource.getWhiteRemoteAddress()));
-
-        return plainAccessResource;
+        return PlainAccessResource.build(plainAccessConfig, remoteAddressStrategyFactory.
+            getRemoteAddressStrategy(plainAccessConfig.getWhiteRemoteAddress()));
     }
 
     public void validate(PlainAccessResource plainAccessResource) {

broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java

@@ -185,6 +185,7 @@ public class BrokerController {
     protected final Broker2Client broker2Client;
     protected final SubscriptionGroupManager subscriptionGroupManager;
     protected final ConsumerIdsChangeListener consumerIdsChangeListener;
+    protected final EndTransactionProcessor endTransactionProcessor;
     private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager();
     protected BrokerOuterAPI brokerOuterAPI;
     protected ScheduledExecutorService scheduledExecutorService;
@@ -317,6 +318,7 @@ public class BrokerController {
         this.queryAssignmentProcessor = new QueryAssignmentProcessor(this);
         this.clientManageProcessor = new ClientManageProcessor(this);
         this.slaveSynchronize = new SlaveSynchronize(this);
+        this.endTransactionProcessor = new EndTransactionProcessor(this);
 
         this.sendThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getSendThreadPoolQueueCapacity());
         this.putThreadPoolQueue = new LinkedBlockingQueue<Runnable>(this.brokerConfig.getPutThreadPoolQueueCapacity());
@@ -1002,8 +1004,8 @@ public class BrokerController {
         /**
          * EndTransactionProcessor
          */
-        this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor);
-        this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), this.endTransactionExecutor);
+        this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
+        this.fastRemotingServer.registerProcessor(RequestCode.END_TRANSACTION, endTransactionProcessor, this.endTransactionExecutor);
 
         /*
          * Default
@@ -1144,6 +1146,14 @@ public class BrokerController {
         return popMessageProcessor;
     }
 
+    public AckMessageProcessor getAckMessageProcessor() {
+        return ackMessageProcessor;
+    }
+
+    public ChangeInvisibleTimeProcessor getChangeInvisibleTimeProcessor() {
+        return changeInvisibleTimeProcessor;
+    }
+
     protected void shutdownBasicService() {
 
         shutdown = true;
@@ -2036,6 +2046,10 @@ public class BrokerController {
         return assignmentManager;
     }
 
+    public ClientManageProcessor getClientManageProcessor() {
+        return clientManageProcessor;
+    }
+
     public SendMessageProcessor getSendMessageProcessor() {
         return sendMessageProcessor;
     }
@@ -2096,6 +2110,10 @@ public class BrokerController {
         return brokerPreOnlineService;
     }
 
+    public EndTransactionProcessor getEndTransactionProcessor() {
+        return endTransactionProcessor;
+    }
+
     public boolean isScheduleServiceStart() {
         return isScheduleServiceStart;
     }

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupEvent.java

@@ -29,5 +29,13 @@ public enum ConsumerGroupEvent {
     /**
      * The group of consumer is registered.
      */
-    REGISTER
+    REGISTER,
+    /**
+     * The client of this consumer is new registered.
+     */
+    CLIENT_REGISTER,
+    /**
+     * The client of this consumer is unregistered.
+     */
+    CLIENT_UNREGISTER
 }

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java

@@ -98,23 +98,24 @@ public class ConsumerGroupInfo {
         return result;
     }
 
-    public void unregisterChannel(final ClientChannelInfo clientChannelInfo) {
+    public boolean unregisterChannel(final ClientChannelInfo clientChannelInfo) {
         ClientChannelInfo old = this.channelInfoTable.remove(clientChannelInfo.getChannel());
         if (old != null) {
             log.info("unregister a consumer[{}] from consumerGroupInfo {}", this.groupName, old.toString());
+            return true;
         }
+        return false;
     }
 
-    public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) {
+    public ClientChannelInfo doChannelCloseEvent(final String remoteAddr, final Channel channel) {
         final ClientChannelInfo info = this.channelInfoTable.remove(channel);
         if (info != null) {
             log.warn(
                 "NETTY EVENT: remove not active channel[{}] from ConsumerGroupInfo groupChannelTable, consumer group: {}",
                 info.toString(), groupName);
-            return true;
         }
 
-        return false;
+        return info;
     }
 
     /**

broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerManager.java

@@ -18,12 +18,15 @@ package org.apache.rocketmq.broker.client;
 
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
 import io.netty.channel.Channel;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
 import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
@@ -40,16 +43,17 @@ public class ConsumerManager {
     private static final long CHANNEL_EXPIRED_TIMEOUT = 1000 * 120;
     private final ConcurrentMap<String, ConsumerGroupInfo> consumerTable =
         new ConcurrentHashMap<String, ConsumerGroupInfo>(1024);
-    private final ConsumerIdsChangeListener consumerIdsChangeListener;
+    private final List<ConsumerIdsChangeListener> consumerIdsChangeListenerList = new CopyOnWriteArrayList<>();
     protected final BrokerStatsManager brokerStatsManager;
 
     public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener) {
-        this.consumerIdsChangeListener = consumerIdsChangeListener;
+        this.consumerIdsChangeListenerList.add(consumerIdsChangeListener);
         this.brokerStatsManager = null;
     }
 
-    public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener, final BrokerStatsManager brokerStatsManager) {
-        this.consumerIdsChangeListener = consumerIdsChangeListener;
+    public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener,
+        final BrokerStatsManager brokerStatsManager) {
+        this.consumerIdsChangeListenerList.add(consumerIdsChangeListener);
         this.brokerStatsManager = brokerStatsManager;
     }
 
@@ -93,18 +97,19 @@ public class ConsumerManager {
         while (it.hasNext()) {
             Entry<String, ConsumerGroupInfo> next = it.next();
             ConsumerGroupInfo info = next.getValue();
-            removed = info.doChannelCloseEvent(remoteAddr, channel);
-            if (removed) {
+            ClientChannelInfo clientChannelInfo = info.doChannelCloseEvent(remoteAddr, channel);
+            if (clientChannelInfo != null) {
+                callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, next.getKey(), clientChannelInfo, info.getSubscribeTopics());
                 if (info.getChannelInfoTable().isEmpty()) {
                     ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey());
                     if (remove != null) {
                         LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}",
                             next.getKey());
-                        this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, next.getKey());
+                        callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, next.getKey());
                     }
                 }
 
-                this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
+                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, next.getKey(), info.getAllChannel());
             }
         }
         return removed;
@@ -113,9 +118,18 @@ public class ConsumerManager {
     public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
         ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
         final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {
+        return registerConsumer(group, clientChannelInfo, consumeType, messageModel, consumeFromWhere, subList,
+            isNotifyConsumerIdsChangedEnable, true);
+    }
+
+    public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
+        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
+        final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable, boolean updateSubscription) {
         long start = System.currentTimeMillis();
         ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
         if (null == consumerGroupInfo) {
+            callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_REGISTER, group, clientChannelInfo,
+                subList.stream().map(SubscriptionData::getTopic).collect(Collectors.toSet()));
             ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
             ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
             consumerGroupInfo = prev != null ? prev : tmp;
@@ -124,18 +138,21 @@ public class ConsumerManager {
         boolean r1 =
             consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,
                 consumeFromWhere);
-        boolean r2 = consumerGroupInfo.updateSubscription(subList);
+        boolean r2 = false;
+        if (updateSubscription) {
+            r2 = consumerGroupInfo.updateSubscription(subList);
+        }
 
         if (r1 || r2) {
             if (isNotifyConsumerIdsChangedEnable) {
-                this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
+                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
             }
         }
         if (null != this.brokerStatsManager) {
             this.brokerStatsManager.incConsumerRegisterTime((int) (System.currentTimeMillis() - start));
         }
 
-        this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);
+        callConsumerIdsChangeListener(ConsumerGroupEvent.REGISTER, group, subList);
 
         return r1 || r2;
     }
@@ -144,17 +161,20 @@ public class ConsumerManager {
         boolean isNotifyConsumerIdsChangedEnable) {
         ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
         if (null != consumerGroupInfo) {
-            consumerGroupInfo.unregisterChannel(clientChannelInfo);
+            boolean removed = consumerGroupInfo.unregisterChannel(clientChannelInfo);
+            if (removed) {
+                callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());
+            }
             if (consumerGroupInfo.getChannelInfoTable().isEmpty()) {
                 ConsumerGroupInfo remove = this.consumerTable.remove(group);
                 if (remove != null) {
                     LOGGER.info("unregister consumer ok, no any connection, and remove consumer group, {}", group);
 
-                    this.consumerIdsChangeListener.handle(ConsumerGroupEvent.UNREGISTER, group);
+                    callConsumerIdsChangeListener(ConsumerGroupEvent.UNREGISTER, group);
                 }
             }
             if (isNotifyConsumerIdsChangedEnable) {
-                this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
+                callConsumerIdsChangeListener(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
             }
         }
     }
@@ -177,6 +197,7 @@ public class ConsumerManager {
                     LOGGER.warn(
                         "SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}",
                         RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group);
+                    callConsumerIdsChangeListener(ConsumerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo, consumerGroupInfo.getSubscribeTopics());
                     RemotingUtil.closeChannel(clientChannelInfo.getChannel());
                     itChannel.remove();
                 }
@@ -204,4 +225,18 @@ public class ConsumerManager {
         }
         return groups;
     }
+
+    public void appendConsumerIdsChangeListener(ConsumerIdsChangeListener listener) {
+        consumerIdsChangeListenerList.add(listener);
+    }
+
+    protected void callConsumerIdsChangeListener(ConsumerGroupEvent event, String group, Object... args) {
+        for (ConsumerIdsChangeListener listener : consumerIdsChangeListenerList) {
+            try {
+                listener.handle(event, group, args);
+            } catch (Throwable t) {
+                LOGGER.error("err when call consumerIdsChangeListener", t);
+            }
+        }
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java

@@ -90,6 +90,9 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen
                 Collection<SubscriptionData> subscriptionDataList = (Collection<SubscriptionData>) args[0];
                 this.brokerController.getConsumerFilterManager().register(group, subscriptionDataList);
                 break;
+            case CLIENT_REGISTER:
+            case CLIENT_UNREGISTER:
+                break;
             default:
                 throw new RuntimeException("Unknown event " + event);
         }

broker/src/main/java/org/apache/rocketmq/broker/client/ProducerChangeListener.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client;
+
+/**
+ * producer manager will call this listener when something happen
+ * <p>
+ * event type: {@link ProducerGroupEvent}
+ */
+public interface ProducerChangeListener {
+
+    void handle(ProducerGroupEvent event, String group, ClientChannelInfo clientChannelInfo);
+}

broker/src/main/java/org/apache/rocketmq/broker/client/ProducerGroupEvent.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client;
+
+public enum ProducerGroupEvent {
+    /**
+     * The group of producer is unregistered.
+     */
+    GROUP_UNREGISTER,
+    /**
+     * The client of this producer is unregistered.
+     */
+    CLIENT_UNREGISTER
+}

broker/src/main/java/org/apache/rocketmq/broker/client/ProducerManager.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.rocketmq.broker.util.PositiveAtomicCounter;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.protocol.body.ProducerInfo;
@@ -44,6 +45,7 @@ public class ProducerManager {
     private final ConcurrentHashMap<String, Channel> clientChannelTable = new ConcurrentHashMap<>();
     protected final BrokerStatsManager brokerStatsManager;
     private PositiveAtomicCounter positiveAtomicCounter = new PositiveAtomicCounter();
+    private final List<ProducerChangeListener> producerChangeListenerList = new CopyOnWriteArrayList<>();
 
     public ProducerManager() {
         this.brokerStatsManager = null;
@@ -94,8 +96,11 @@ public class ProducerManager {
     }
 
     public void scanNotActiveChannel() {
-        for (final Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>> entry : this.groupChannelTable
-                .entrySet()) {
+        Iterator<Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>>> iterator = this.groupChannelTable.entrySet().iterator();
+
+        while (iterator.hasNext()) {
+            Map.Entry<String, ConcurrentHashMap<Channel, ClientChannelInfo>> entry = iterator.next();
+
             final String group = entry.getKey();
             final ConcurrentHashMap<Channel, ClientChannelInfo> chlMap = entry.getValue();
 
@@ -112,9 +117,16 @@ public class ProducerManager {
                     log.warn(
                             "ProducerManager#scanNotActiveChannel: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}",
                             RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group);
+                    callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, info);
                     RemotingUtil.closeChannel(info.getChannel());
                 }
             }
+
+            if (chlMap.isEmpty()) {
+                log.warn("SCAN: remove expired channel from ProducerManager groupChannelTable, all clear, group={}", group);
+                iterator.remove();
+                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);
+            }
         }
     }
 
@@ -134,6 +146,14 @@ public class ProducerManager {
                     log.info(
                             "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}",
                             clientChannelInfo.toString(), remoteAddr, group);
+                    callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);
+                    if (clientChannelInfoTable.isEmpty()) {
+                        ConcurrentHashMap<Channel, ClientChannelInfo> oldGroupTable = this.groupChannelTable.remove(group);
+                        if (oldGroupTable != null) {
+                            log.info("unregister a producer group[{}] from groupChannelTable", group);
+                            callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);
+                        }
+                    }
                 }
 
             }
@@ -172,10 +192,12 @@ public class ProducerManager {
             if (old != null) {
                 log.info("unregister a producer[{}] from groupChannelTable {}", group,
                         clientChannelInfo.toString());
+                callProducerChangeListener(ProducerGroupEvent.CLIENT_UNREGISTER, group, clientChannelInfo);
             }
 
             if (channelTable.isEmpty()) {
                 this.groupChannelTable.remove(group);
+                callProducerChangeListener(ProducerGroupEvent.GROUP_UNREGISTER, group, null);
                 log.info("unregister a producer group[{}] from groupChannelTable", group);
             }
         }
@@ -224,4 +246,19 @@ public class ProducerManager {
     public Channel findChannel(String clientId) {
         return clientChannelTable.get(clientId);
     }
+
+    private void callProducerChangeListener(ProducerGroupEvent event, String group,
+        ClientChannelInfo clientChannelInfo) {
+        for (ProducerChangeListener listener : producerChangeListenerList) {
+            try {
+                listener.handle(event, group, clientChannelInfo);
+            } catch (Throwable t) {
+                log.error("err when call producerChangeListener", t);
+            }
+        }
+    }
+
+    public void appendProducerChangeListener(ProducerChangeListener producerChangeListener) {
+        producerChangeListenerList.add(producerChangeListener);
+    }
 }

broker/src/main/java/org/apache/rocketmq/broker/processor/AbstractSendMessageProcessor.java

@@ -18,7 +18,6 @@ package org.apache.rocketmq.broker.processor;
 
 import io.netty.channel.ChannelHandlerContext;
 import java.net.SocketAddress;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -32,15 +31,10 @@ import org.apache.rocketmq.broker.mqtrace.SendMessageContext;
 import org.apache.rocketmq.broker.mqtrace.SendMessageHook;
 import org.apache.rocketmq.common.BrokerConfig;
 import org.apache.rocketmq.common.MQVersion;
-import org.apache.rocketmq.common.UtilAll;
-import org.apache.rocketmq.common.message.MessageExt;
-import org.apache.rocketmq.common.message.MessageType;
-import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
-import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
-import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.common.MixAll;
 import org.apache.rocketmq.common.TopicConfig;
 import org.apache.rocketmq.common.TopicFilterType;
+import org.apache.rocketmq.common.UtilAll;
 import org.apache.rocketmq.common.constant.DBMsgConstants;
 import org.apache.rocketmq.common.constant.LoggerName;
 import org.apache.rocketmq.common.constant.PermName;
@@ -48,21 +42,24 @@ import org.apache.rocketmq.common.help.FAQUrl;
 import org.apache.rocketmq.common.message.MessageAccessor;
 import org.apache.rocketmq.common.message.MessageConst;
 import org.apache.rocketmq.common.message.MessageDecoder;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageExtBrokerInner;
+import org.apache.rocketmq.common.message.MessageType;
 import org.apache.rocketmq.common.protocol.NamespaceUtil;
-import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.protocol.ResponseCode;
+import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader;
 import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader;
-import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeaderV2;
 import org.apache.rocketmq.common.protocol.header.SendMessageResponseHeader;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.common.sysflag.TopicSysFlag;
+import org.apache.rocketmq.common.topic.TopicValidator;
 import org.apache.rocketmq.logging.InternalLogger;
 import org.apache.rocketmq.logging.InternalLoggerFactory;
 import org.apache.rocketmq.remoting.common.RemotingHelper;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
 import org.apache.rocketmq.remoting.netty.NettyRequestProcessor;
 import org.apache.rocketmq.remoting.protocol.RemotingCommand;
-import org.apache.rocketmq.common.message.MessageExtBrokerInner;
 import org.apache.rocketmq.store.PutMessageResult;
 import org.apache.rocketmq.store.stats.BrokerStatsManager;
 
@@ -574,102 +571,8 @@ public abstract class AbstractSendMessageProcessor implements NettyRequestProces
         }
     }
 
-    protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request)
-        throws RemotingCommandException {
-
-        SendMessageRequestHeaderV2 requestHeaderV2 = null;
-        SendMessageRequestHeader requestHeader = null;
-        switch (request.getCode()) {
-            case RequestCode.SEND_BATCH_MESSAGE:
-            case RequestCode.SEND_MESSAGE_V2:
-                requestHeaderV2 =
-                        (SendMessageRequestHeaderV2) request
-                                .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
-            case RequestCode.SEND_MESSAGE:
-                if (null == requestHeaderV2) {
-                    requestHeader =
-                        (SendMessageRequestHeader) request
-                            .decodeCommandCustomHeader(SendMessageRequestHeader.class);
-                } else {
-                    requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2);
-                }
-            default:
-                break;
-        }
-        return requestHeader;
-    }
-
-    static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request)
-        throws RemotingCommandException {
-        SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
-        HashMap<String, String> fields = request.getExtFields();
-        if (fields == null) {
-            throw new RemotingCommandException("the ext fields is null");
-        }
-
-        String s = fields.get("a");
-        checkNotNull(s, "the custom field <a> is null");
-        r.setA(s);
-
-        s = fields.get("b");
-        checkNotNull(s, "the custom field <b> is null");
-        r.setB(s);
-
-        s = fields.get("c");
-        checkNotNull(s, "the custom field <c> is null");
-        r.setC(s);
-
-        s = fields.get("d");
-        checkNotNull(s, "the custom field <d> is null");
-        r.setD(Integer.parseInt(s));
-
-        s = fields.get("e");
-        checkNotNull(s, "the custom field <e> is null");
-        r.setE(Integer.parseInt(s));
-
-        s = fields.get("f");
-        checkNotNull(s, "the custom field <f> is null");
-        r.setF(Integer.parseInt(s));
-
-        s = fields.get("g");
-        checkNotNull(s, "the custom field <g> is null");
-        r.setG(Long.parseLong(s));
-
-        s = fields.get("h");
-        checkNotNull(s, "the custom field <h> is null");
-        r.setH(Integer.parseInt(s));
-
-        s = fields.get("i");
-        if (s != null) {
-            r.setI(s);
-        }
-
-        s = fields.get("j");
-        if (s != null) {
-            r.setJ(Integer.parseInt(s));
-        }
-
-        s = fields.get("k");
-        if (s != null) {
-            r.setK(Boolean.parseBoolean(s));
-        }
-
-        s = fields.get("l");
-        if (s != null) {
-            r.setL(Integer.parseInt(s));
-        }
-
-        s = fields.get("m");
-        if (s != null) {
-            r.setM(Boolean.parseBoolean(s));
-        }
-        return r;
-    }
-
-    private static void checkNotNull(String s, String msg) throws RemotingCommandException {
-        if (s == null) {
-            throw new RemotingCommandException(msg);
-        }
+    protected SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException {
+        return SendMessageRequestHeader.parseRequestHeader(request);
     }
 
     protected int randomQueueId(int writeQueueNums) {

broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java

@@ -179,6 +179,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor {
         ck.setTopic(requestHeader.getTopic());
         ck.setQueueId((byte) queueId);
         ck.addDiff(0);
+        ck.setBrokerName(brokerName);
 
         msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset));
         msgInner.setQueueId(reviveQid);

broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java

@@ -33,7 +33,6 @@ import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.rocketmq.broker.BrokerController;
-import org.apache.rocketmq.broker.client.ConsumerGroupInfo;
 import org.apache.rocketmq.broker.filter.ConsumerFilterData;
 import org.apache.rocketmq.broker.filter.ConsumerFilterManager;
 import org.apache.rocketmq.broker.filter.ExpressionMessageFilter;
@@ -251,7 +250,7 @@ public class PopMessageProcessor implements NettyRequestProcessor {
         }
 
         if (requestHeader.isTimeoutTooMuch()) {
-            response.setCode(POLLING_TIMEOUT);
+            response.setCode(ResponseCode.POLLING_TIMEOUT);
             response.setRemark(String.format("the broker[%s] poping message is timeout too much",
                 this.brokerController.getBrokerConfig().getBrokerIP1()));
             return response;
@@ -304,14 +303,6 @@ public class PopMessageProcessor implements NettyRequestProcessor {
                 requestHeader.getConsumerGroup(), FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)));
             return response;
         }
-        ConsumerGroupInfo consumerGroupInfo =
-            this.brokerController.getConsumerManager().getConsumerGroupInfo(requestHeader.getConsumerGroup());
-        if (null == consumerGroupInfo) {
-            POP_LOGGER.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup());
-            response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST);
-            response.setRemark("the consumer's group info not exist" + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC));
-            return response;
-        }
 
         if (!subscriptionGroupConfig.isConsumeEnable()) {
             response.setCode(ResponseCode.NO_PERMISSION);
@@ -463,6 +454,8 @@ public class PopMessageProcessor implements NettyRequestProcessor {
                     response = null;
                 }
                 break;
+            case ResponseCode.POLLING_TIMEOUT:
+                return response;
             default:
                 assert false;
         }

broker/src/main/java/org/apache/rocketmq/broker/transaction/TransactionalMessageCheckService.java

@@ -42,8 +42,8 @@ public class TransactionalMessageCheckService extends ServiceThread {
     @Override
     public void run() {
         log.info("Start transaction check service thread!");
-        long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval();
         while (!this.isStopped()) {
+            long checkInterval = brokerController.getBrokerConfig().getTransactionCheckInterval();
             this.waitForRunning(checkInterval);
         }
         log.info("End transaction check service thread!");

broker/src/test/java/org/apache/rocketmq/broker/client/ConsumerManagerScannerTest.java

@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.broker.client;
+
+import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
+import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType;
+import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;
+import org.apache.rocketmq.remoting.protocol.LanguageCode;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumerManagerScannerTest {
+    private ConsumerManager consumerManager;
+    private String group = "FooBar";
+    private String clientId = "clientId";
+    private ClientChannelInfo clientInfo;
+    private Map<ConsumerGroupEvent, List<ConsumerIdsChangeListenerData>> groupEventListMap = new HashMap<>();
+
+    @Mock
+    private Channel channel;
+
+    @Before
+    public void init() {
+        clientInfo = new ClientChannelInfo(channel, clientId, LanguageCode.JAVA, 0);
+
+        consumerManager = new ConsumerManager(new ConsumerIdsChangeListener() {
+            @Override
+            public void handle(ConsumerGroupEvent event, String group, Object... args) {
+                groupEventListMap.compute(event, (eventKey, dataListVal) -> {
+                    if (dataListVal == null) {
+                        dataListVal = new ArrayList<>();
+                    }
+                    dataListVal.add(new ConsumerIdsChangeListenerData(event, group, args));
+                    return dataListVal;
+                });
+            }
+
+            @Override
+            public void shutdown() {
+
+            }
+        });
+    }
+
+    private static class ConsumerIdsChangeListenerData {
+        private ConsumerGroupEvent event;
+        private String group;
+        private Object[] args;
+
+        public ConsumerIdsChangeListenerData(ConsumerGroupEvent event, String group, Object[] args) {
+            this.event = event;
+            this.group = group;
+            this.args = args;
+        }
+    }
+
+    @Test
+    public void testClientUnregisterEventInDoChannelCloseEvent() {
+        assertThat(consumerManager.registerConsumer(
+            group,
+            clientInfo,
+            ConsumeType.CONSUME_PASSIVELY,
+            MessageModel.CLUSTERING,
+            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+            new HashSet<>(),
+            false
+        )).isTrue();
+
+        consumerManager.doChannelCloseEvent("remoteAddr", channel);
+
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);
+        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];
+        assertThat(clientChannelInfo).isSameAs(clientInfo);
+    }
+
+    @Test
+    public void testClientUnregisterEventInUnregisterConsumer() {
+        assertThat(consumerManager.registerConsumer(
+            group,
+            clientInfo,
+            ConsumeType.CONSUME_PASSIVELY,
+            MessageModel.CLUSTERING,
+            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+            new HashSet<>(),
+            false
+        )).isTrue();
+
+        consumerManager.unregisterConsumer(group, clientInfo, false);
+
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);
+        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];
+        assertThat(clientChannelInfo).isSameAs(clientInfo);
+    }
+
+    @Test
+    public void testClientUnregisterEventInScanNotActiveChannel() {
+        assertThat(consumerManager.registerConsumer(
+            group,
+            clientInfo,
+            ConsumeType.CONSUME_PASSIVELY,
+            MessageModel.CLUSTERING,
+            ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET,
+            new HashSet<>(),
+            false
+        )).isTrue();
+        clientInfo.setLastUpdateTimestamp(0);
+        when(channel.close()).thenReturn(mock(ChannelFuture.class));
+
+        consumerManager.scanNotActiveChannel();
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).size()).isEqualTo(1);
+        assertThat(groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0]).isInstanceOf(ClientChannelInfo.class);
+        ClientChannelInfo clientChannelInfo = (ClientChannelInfo) groupEventListMap.get(ConsumerGroupEvent.CLIENT_UNREGISTER).get(0).args[0];
+        assertThat(clientChannelInfo).isSameAs(clientInfo);
+    }
+}

broker/src/test/java/org/apache/rocketmq/broker/client/ProducerManagerTest.java

@@ -21,6 +21,7 @@ import io.netty.channel.ChannelFuture;
 import java.lang.reflect.Field;
 import java.util.Map;
 
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.rocketmq.remoting.protocol.LanguageCode;
 import org.junit.Before;
 import org.junit.Test;
@@ -50,6 +51,20 @@ public class ProducerManagerTest {
     @Test
     public void scanNotActiveChannel() throws Exception {
         producerManager.registerProducer(group, clientInfo);
+        AtomicReference<String> groupRef = new AtomicReference<>();
+        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();
+        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {
+            switch (event) {
+                case GROUP_UNREGISTER:
+                    groupRef.set(group);
+                    break;
+                case CLIENT_UNREGISTER:
+                    clientChannelInfoRef.set(clientChannelInfo);
+                    break;
+                default:
+                    break;
+            }
+        });
         assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull();
         assertThat(producerManager.findChannel("clientId")).isNotNull();
         Field field = ProducerManager.class.getDeclaredField("CHANNEL_EXPIRED_TIMEOUT");
@@ -58,17 +73,35 @@ public class ProducerManagerTest {
         clientInfo.setLastUpdateTimestamp(System.currentTimeMillis() - CHANNEL_EXPIRED_TIMEOUT - 10);
         when(channel.close()).thenReturn(mock(ChannelFuture.class));
         producerManager.scanNotActiveChannel();
-        assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull();
+        assertThat(producerManager.getGroupChannelTable().get(group)).isNull();
+        assertThat(groupRef.get()).isEqualTo(group);
+        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);
         assertThat(producerManager.findChannel("clientId")).isNull();
     }
 
     @Test
     public void doChannelCloseEvent() throws Exception {
         producerManager.registerProducer(group, clientInfo);
+        AtomicReference<String> groupRef = new AtomicReference<>();
+        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();
+        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {
+            switch (event) {
+                case GROUP_UNREGISTER:
+                    groupRef.set(group);
+                    break;
+                case CLIENT_UNREGISTER:
+                    clientChannelInfoRef.set(clientChannelInfo);
+                    break;
+                default:
+                    break;
+            }
+        });
         assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNotNull();
         assertThat(producerManager.findChannel("clientId")).isNotNull();
         producerManager.doChannelCloseEvent("127.0.0.1", channel);
-        assertThat(producerManager.getGroupChannelTable().get(group).get(channel)).isNull();
+        assertThat(producerManager.getGroupChannelTable().get(group)).isNull();
+        assertThat(groupRef.get()).isEqualTo(group);
+        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);
         assertThat(producerManager.findChannel("clientId")).isNull();
     }
 
@@ -86,6 +119,20 @@ public class ProducerManagerTest {
     @Test
     public void unregisterProducer() throws Exception {
         producerManager.registerProducer(group, clientInfo);
+        AtomicReference<String> groupRef = new AtomicReference<>();
+        AtomicReference<ClientChannelInfo> clientChannelInfoRef = new AtomicReference<>();
+        producerManager.appendProducerChangeListener((event, group, clientChannelInfo) -> {
+            switch (event) {
+                case GROUP_UNREGISTER:
+                    groupRef.set(group);
+                    break;
+                case CLIENT_UNREGISTER:
+                    clientChannelInfoRef.set(clientChannelInfo);
+                    break;
+                default:
+                    break;
+            }
+        });
         Map<Channel, ClientChannelInfo> channelMap = producerManager.getGroupChannelTable().get(group);
         assertThat(channelMap).isNotNull();
         assertThat(channelMap.get(channel)).isEqualTo(clientInfo);
@@ -95,6 +142,8 @@ public class ProducerManagerTest {
         producerManager.unregisterProducer(group, clientInfo);
         channelMap = producerManager.getGroupChannelTable().get(group);
         channel1 = producerManager.findChannel("clientId");
+        assertThat(groupRef.get()).isEqualTo(group);
+        assertThat(clientChannelInfoRef.get()).isSameAs(clientInfo);
         assertThat(channelMap).isNull();
         assertThat(channel1).isNull();
 

broker/src/test/java/org/apache/rocketmq/broker/processor/PopMessageProcessorTest.java

@@ -110,16 +110,6 @@ public class PopMessageProcessorTest {
         assertThat(response.getRemark()).contains("topic[" + topic + "] not exist");
     }
 
-    @Test
-    public void testProcessRequest_SubNotExist() throws RemotingCommandException {
-        brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo, false);
-        final RemotingCommand request = createPopMsgCommand();
-        RemotingCommand response = popMessageProcessor.processRequest(handlerContext, request);
-        assertThat(response).isNotNull();
-        assertThat(response.getCode()).isEqualTo(ResponseCode.SUBSCRIPTION_NOT_EXIST);
-        assertThat(response.getRemark()).contains("consumer's group info not exist");
-    }
-
     @Test
     public void testProcessRequest_Found() throws RemotingCommandException {
         GetMessageResult getMessageResult = createGetMessageResult(1);

client/src/main/java/org/apache/rocketmq/client/ClientConfig.java

@@ -35,6 +35,8 @@ import org.apache.rocketmq.remoting.protocol.RequestType;
  */
 public class ClientConfig {
     public static final String SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY = "com.rocketmq.sendMessageWithVIPChannel";
+    public static final String DECODE_READ_BODY = "com.rocketmq.read.body";
+    public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body";
     private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses();
     private String clientIP = RemotingUtil.getLocalAddress();
     private String instanceName = System.getProperty("rocketmq.client.name", "DEFAULT");
@@ -58,6 +60,8 @@ public class ClientConfig {
     private long pullTimeDelayMillsWhenException = 1000;
     private boolean unitMode = false;
     private String unitName;
+    private boolean decodeReadBody = Boolean.parseBoolean(System.getProperty(DECODE_READ_BODY, "true"));
+    private boolean decodeDecompressBody = Boolean.parseBoolean(System.getProperty(DECODE_DECOMPRESS_BODY, "true"));
     private boolean vipChannelEnabled = Boolean.parseBoolean(System.getProperty(SEND_MESSAGE_WITH_VIP_CHANNEL_PROPERTY, "false"));
 
     private boolean useTLS = TlsSystemConfig.tlsEnable;
@@ -172,6 +176,8 @@ public class ClientConfig {
         this.namespace = cc.namespace;
         this.language = cc.language;
         this.mqClientApiTimeout = cc.mqClientApiTimeout;
+        this.decodeReadBody = cc.decodeReadBody;
+        this.decodeDecompressBody = cc.decodeDecompressBody;
         this.enableStreamRequestType = cc.enableStreamRequestType;
     }
 
@@ -192,6 +198,8 @@ public class ClientConfig {
         cc.namespace = namespace;
         cc.language = language;
         cc.mqClientApiTimeout = mqClientApiTimeout;
+        cc.decodeReadBody = decodeReadBody;
+        cc.decodeDecompressBody = decodeDecompressBody;
         cc.enableStreamRequestType = enableStreamRequestType;
         return cc;
     }
@@ -293,6 +301,22 @@ public class ClientConfig {
         this.language = language;
     }
 
+    public boolean isDecodeReadBody() {
+        return decodeReadBody;
+    }
+
+    public void setDecodeReadBody(boolean decodeReadBody) {
+        this.decodeReadBody = decodeReadBody;
+    }
+
+    public boolean isDecodeDecompressBody() {
+        return decodeDecompressBody;
+    }
+
+    public void setDecodeDecompressBody(boolean decodeDecompressBody) {
+        this.decodeDecompressBody = decodeDecompressBody;
+    }
+
     public String getNamespace() {
         if (namespaceInitialized) {
             return namespace;
@@ -347,6 +371,7 @@ public class ClientConfig {
             + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval
             + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + ", unitMode=" + unitMode + ", unitName=" + unitName + ", vipChannelEnabled="
             + vipChannelEnabled + ", useTLS=" + useTLS + ", language=" + language.name() + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout
+            + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody
             + ", enableStreamRequestType=" + enableStreamRequestType + "]";
     }
 }

client/src/main/java/org/apache/rocketmq/client/exception/MQClientException.java

@@ -37,6 +37,13 @@ public class MQClientException extends Exception {
         this.errorMessage = errorMessage;
     }
 
+    public MQClientException(int responseCode, String errorMessage, Throwable cause) {
+        super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + "  DESC: "
+            + errorMessage), cause);
+        this.responseCode = responseCode;
+        this.errorMessage = errorMessage;
+    }
+
     public int getResponseCode() {
         return responseCode;
     }

client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java

@@ -31,6 +31,7 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.rocketmq.client.ClientConfig;
+import org.apache.rocketmq.client.common.ClientErrorCode;
 import org.apache.rocketmq.client.consumer.AckCallback;
 import org.apache.rocketmq.client.consumer.AckResult;
 import org.apache.rocketmq.client.consumer.AckStatus;
@@ -735,7 +736,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
         }
     }
 
-    private SendResult processSendResponse(
+    protected SendResult processSendResponse(
         final String brokerName,
         final Message msg,
         final RemotingCommand response,
@@ -853,9 +854,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                     }
                 } else {
                     if (!responseFuture.isSendRequestOK()) {
-                        popCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
+                        popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
                     } else if (responseFuture.isTimeout()) {
-                        popCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
+                        popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
                             responseFuture.getCause()));
                     } else {
                         popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
@@ -892,9 +893,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                     }
                 } else {
                     if (!responseFuture.isSendRequestOK()) {
-                        ackCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
+                        ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
                     } else if (responseFuture.isTimeout()) {
-                        ackCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
+                        ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
                             responseFuture.getCause()));
                     } else {
                         ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause()));
@@ -938,9 +939,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                     }
                 } else {
                     if (!responseFuture.isSendRequestOK()) {
-                        ackCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
+                        ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
                     } else if (responseFuture.isTimeout()) {
-                        ackCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
+                        ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
                             responseFuture.getCause()));
                     } else {
                         ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
@@ -970,9 +971,9 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
                     }
                 } else {
                     if (!responseFuture.isSendRequestOK()) {
-                        pullCallback.onException(new MQClientException("send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
+                        pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause()));
                     } else if (responseFuture.isTimeout()) {
-                        pullCallback.onException(new MQClientException("wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
+                        pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request,
                             responseFuture.getCause()));
                     } else {
                         pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause()));
@@ -1029,7 +1030,11 @@ public class MQClientAPIImpl implements NameServerUpdateCallback {
             case ResponseCode.SUCCESS:
                 popStatus = PopStatus.FOUND;
                 ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody());
-                msgFoundList = MessageDecoder.decodes(byteBuffer);
+                msgFoundList = MessageDecoder.decodesBatch(
+                    byteBuffer,
+                    clientConfig.isDecodeReadBody(),
+                    clientConfig.isDecodeDecompressBody(),
+                    true);
                 break;
             case ResponseCode.POLLING_FULL:
                 popStatus = PopStatus.POLLING_FULL;

client/src/main/java/org/apache/rocketmq/client/impl/consumer/PullAPIWrapper.java

@@ -78,7 +78,12 @@ public class PullAPIWrapper {
         this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId());
         if (PullStatus.FOUND == pullResult.getPullStatus()) {
             ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary());
-            List<MessageExt> msgList = MessageDecoder.decodes(byteBuffer);
+            List<MessageExt> msgList = MessageDecoder.decodesBatch(
+                byteBuffer,
+                this.mQClientFactory.getClientConfig().isDecodeReadBody(),
+                this.mQClientFactory.getClientConfig().isDecodeDecompressBody(),
+                true
+            );
 
             boolean needDecodeInnerMessage = false;
             for (MessageExt messageExt: msgList) {

common/pom.xml

@@ -52,5 +52,14 @@
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.7</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
     </dependencies>
 </project>

common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java

@@ -44,4 +44,6 @@ public class LoggerName {
     public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop";
     public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover";
     public static final String STDOUT_LOGGER_NAME = "STDOUT";
+    public static final String PROXY_LOGGER_NAME = "RocketmqProxy";
+    public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark";
 }

common/src/main/java/org/apache/rocketmq/common/consumer/ReceiptHandle.java

@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.consumer;
+
+import java.util.Arrays;
+import java.util.List;
+import org.apache.rocketmq.common.KeyBuilder;
+import org.apache.rocketmq.common.message.MessageConst;
+
+public class ReceiptHandle {
+    private static final String SEPARATOR = MessageConst.KEY_SEPARATOR;
+    public static final String NORMAL_TOPIC = "0";
+    public static final String RETRY_TOPIC = "1";
+    private final long startOffset;
+    private final long retrieveTime;
+    private final long invisibleTime;
+    private final long nextVisibleTime;
+    private final int reviveQueueId;
+    private final String topicType;
+    private final String brokerName;
+    private final int queueId;
+    private final long offset;
+    private final long commitLogOffset;
+    private final String receiptHandle;
+
+    public String encode() {
+        return startOffset + SEPARATOR + retrieveTime + SEPARATOR + invisibleTime + SEPARATOR + reviveQueueId
+            + SEPARATOR + topicType + SEPARATOR + brokerName + SEPARATOR + queueId + SEPARATOR + offset + SEPARATOR
+            + commitLogOffset;
+    }
+
+    public boolean isExpired() {
+        return nextVisibleTime <= System.currentTimeMillis();
+    }
+
+    public static ReceiptHandle decode(String receiptHandle) {
+        List<String> dataList = Arrays.asList(receiptHandle.split(SEPARATOR));
+        if (dataList.size() < 8) {
+            throw new IllegalArgumentException("Parse failed, dataList size " + dataList.size());
+        }
+        long startOffset = Long.parseLong(dataList.get(0));
+        long retrieveTime = Long.parseLong(dataList.get(1));
+        long invisibleTime = Long.parseLong(dataList.get(2));
+        int reviveQueueId = Integer.parseInt(dataList.get(3));
+        String topicType = dataList.get(4);
+        String brokerName = dataList.get(5);
+        int queueId = Integer.parseInt(dataList.get(6));
+        long offset = Long.parseLong(dataList.get(7));
+        long commitLogOffset = -1L;
+        if (dataList.size() >= 9) {
+            commitLogOffset = Long.parseLong(dataList.get(8));
+        }
+
+        return new ReceiptHandleBuilder()
+            .startOffset(startOffset)
+            .retrieveTime(retrieveTime)
+            .invisibleTime(invisibleTime)
+            .reviveQueueId(reviveQueueId)
+            .topicType(topicType)
+            .brokerName(brokerName)
+            .queueId(queueId)
+            .offset(offset)
+            .commitLogOffset(commitLogOffset)
+            .receiptHandle(receiptHandle).build();
+    }
+
+    ReceiptHandle(final long startOffset, final long retrieveTime, final long invisibleTime, final long nextVisibleTime,
+        final int reviveQueueId, final String topicType, final String brokerName, final int queueId, final long offset,
+        final long commitLogOffset, final String receiptHandle) {
+        this.startOffset = startOffset;
+        this.retrieveTime = retrieveTime;
+        this.invisibleTime = invisibleTime;
+        this.nextVisibleTime = nextVisibleTime;
+        this.reviveQueueId = reviveQueueId;
+        this.topicType = topicType;
+        this.brokerName = brokerName;
+        this.queueId = queueId;
+        this.offset = offset;
+        this.commitLogOffset = commitLogOffset;
+        this.receiptHandle = receiptHandle;
+    }
+
+    public static class ReceiptHandleBuilder {
+        private long startOffset;
+        private long retrieveTime;
+        private long invisibleTime;
+        private int reviveQueueId;
+        private String topicType;
+        private String brokerName;
+        private int queueId;
+        private long offset;
+        private long commitLogOffset;
+        private String receiptHandle;
+
+        ReceiptHandleBuilder() {
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder startOffset(final long startOffset) {
+            this.startOffset = startOffset;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder retrieveTime(final long retrieveTime) {
+            this.retrieveTime = retrieveTime;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder invisibleTime(final long invisibleTime) {
+            this.invisibleTime = invisibleTime;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder reviveQueueId(final int reviveQueueId) {
+            this.reviveQueueId = reviveQueueId;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder topicType(final String topicType) {
+            this.topicType = topicType;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder brokerName(final String brokerName) {
+            this.brokerName = brokerName;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder queueId(final int queueId) {
+            this.queueId = queueId;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder offset(final long offset) {
+            this.offset = offset;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder commitLogOffset(final long commitLogOffset) {
+            this.commitLogOffset = commitLogOffset;
+            return this;
+        }
+
+        public ReceiptHandle.ReceiptHandleBuilder receiptHandle(final String receiptHandle) {
+            this.receiptHandle = receiptHandle;
+            return this;
+        }
+
+        public ReceiptHandle build() {
+            return new ReceiptHandle(this.startOffset, this.retrieveTime, this.invisibleTime, this.retrieveTime + this.invisibleTime,
+                this.reviveQueueId, this.topicType, this.brokerName, this.queueId, this.offset, this.commitLogOffset, this.receiptHandle);
+        }
+
+        @Override
+        public String toString() {
+            return "ReceiptHandle.ReceiptHandleBuilder(startOffset=" + this.startOffset + ", retrieveTime=" + this.retrieveTime + ", invisibleTime=" + this.invisibleTime + ", reviveQueueId=" + this.reviveQueueId + ", topic=" + this.topicType + ", brokerName=" + this.brokerName + ", queueId=" + this.queueId + ", offset=" + this.offset + ", commitLogOffset=" + this.commitLogOffset + ", receiptHandle=" + this.receiptHandle + ")";
+        }
+    }
+
+    public static ReceiptHandle.ReceiptHandleBuilder builder() {
+        return new ReceiptHandle.ReceiptHandleBuilder();
+    }
+
+    public long getStartOffset() {
+        return this.startOffset;
+    }
+
+    public long getRetrieveTime() {
+        return this.retrieveTime;
+    }
+
+    public long getInvisibleTime() {
+        return this.invisibleTime;
+    }
+
+    public long getNextVisibleTime() {
+        return this.nextVisibleTime;
+    }
+
+    public int getReviveQueueId() {
+        return this.reviveQueueId;
+    }
+
+    public String getTopicType() {
+        return this.topicType;
+    }
+
+    public String getBrokerName() {
+        return this.brokerName;
+    }
+
+    public int getQueueId() {
+        return this.queueId;
+    }
+
+    public long getOffset() {
+        return this.offset;
+    }
+
+    public long getCommitLogOffset() {
+        return commitLogOffset;
+    }
+
+    public String getReceiptHandle() {
+        return this.receiptHandle;
+    }
+
+    public boolean isRetryTopic() {
+        return RETRY_TOPIC.equals(topicType);
+    }
+
+    public String getRealTopic(String topic, String groupName) {
+        if (isRetryTopic()) {
+            return KeyBuilder.buildPopRetryTopic(topic, groupName);
+        }
+        return topic;
+    }
+}

common/src/main/java/org/apache/rocketmq/common/message/MessageBatch.java

@@ -39,7 +39,7 @@ public class MessageBatch extends Message implements Iterable<Message> {
         return messages.iterator();
     }
 
-    public static MessageBatch generateFromList(Collection<Message> messages) {
+    public static MessageBatch generateFromList(Collection<? extends Message> messages) {
         assert messages != null;
         assert messages.size() > 0;
         List<Message> messageList = new ArrayList<Message>(messages.size());

common/src/main/java/org/apache/rocketmq/common/message/MessageConst.java

@@ -65,6 +65,10 @@ public class MessageConst {
     public static final String PROPERTY_REDIRECT = "REDIRECT";
     public static final String PROPERTY_INNER_MULTI_DISPATCH = "INNER_MULTI_DISPATCH";
     public static final String PROPERTY_INNER_MULTI_QUEUE_OFFSET = "INNER_MULTI_QUEUE_OFFSET";
+    public static final String PROPERTY_TRACE_CONTEXT = "TRACE_CONTEXT";
+    public static final String PROPERTY_TIMER_DELAY_SEC = "TIMER_DELAY_SEC";
+    public static final String PROPERTY_TIMER_DELIVER_MS = "TIMER_DELIVER_MS";
+    public static final String PROPERTY_BORN_HOST = "__BORNHOST";
 
     /**
      * property which name starts with "__RMQ.TRANSIENT." is called transient one that will not stored in broker disks.
@@ -123,5 +127,6 @@ public class MessageConst {
         STRING_HASH_SET.add(PROPERTY_CLUSTER);
         STRING_HASH_SET.add(PROPERTY_MESSAGE_TYPE);
         STRING_HASH_SET.add(PROPERTY_INNER_MULTI_QUEUE_OFFSET);
+        STRING_HASH_SET.add(PROPERTY_BORN_HOST);
     }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/AckMessageRequestHeader.java

@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -80,6 +81,12 @@ public class AckMessageRequestHeader implements CommandCustomHeader {
 
     @Override
     public String toString() {
-        return topic + "," + this.consumerGroup + "," + this.queueId + "," + this.offset + "," + this.extraInfo;
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("extraInfo", extraInfo)
+            .add("offset", offset)
+            .toString();
     }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/ChangeInvisibleTimeRequestHeader.java

@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -94,4 +95,14 @@ public class ChangeInvisibleTimeRequestHeader implements CommandCustomHeader {
         this.queueId = queueId;
     }
 
+    @Override public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("extraInfo", extraInfo)
+            .add("offset", offset)
+            .add("invisibleTime", invisibleTime)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -76,4 +77,15 @@ public class CheckTransactionStateRequestHeader implements CommandCustomHeader {
     public void setOffsetMsgId(String offsetMsgId) {
         this.offsetMsgId = offsetMsgId;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("tranStateTableOffset", tranStateTableOffset)
+            .add("commitLogOffset", commitLogOffset)
+            .add("msgId", msgId)
+            .add("transactionId", transactionId)
+            .add("offsetMsgId", offsetMsgId)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -68,4 +69,14 @@ public class CloneGroupOffsetRequestHeader implements CommandCustomHeader {
     public void setOffline(boolean offline) {
         this.offline = offline;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("srcGroup", srcGroup)
+            .add("destGroup", destGroup)
+            .add("topic", topic)
+            .add("offline", offline)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -97,4 +98,17 @@ public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomH
     public void setGroupSysFlag(Integer groupSysFlag) {
         this.groupSysFlag = groupSysFlag;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("clientId", clientId)
+            .add("msgId", msgId)
+            .add("brokerName", brokerName)
+            .add("topic", topic)
+            .add("topicSysFlag", topicSysFlag)
+            .add("groupSysFlag", groupSysFlag)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -98,7 +99,14 @@ public class ConsumerSendMsgBackRequestHeader implements CommandCustomHeader {
 
     @Override
     public String toString() {
-        return "ConsumerSendMsgBackRequestHeader [group=" + group + ", originTopic=" + originTopic + ", originMsgId=" + originMsgId
-            + ", delayLevel=" + delayLevel + ", unitMode=" + unitMode + ", maxReconsumeTimes=" + maxReconsumeTimes + "]";
+        return MoreObjects.toStringHelper(this)
+            .add("offset", offset)
+            .add("group", group)
+            .add("delayLevel", delayLevel)
+            .add("originMsgId", originMsgId)
+            .add("originTopic", originTopic)
+            .add("unitMode", unitMode)
+            .add("maxReconsumeTimes", maxReconsumeTimes)
+            .toString();
     }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateAccessConfigRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -110,4 +111,18 @@ public class CreateAccessConfigRequestHeader implements CommandCustomHeader {
     public void setGroupPerms(String groupPerms) {
         this.groupPerms = groupPerms;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("accessKey", accessKey)
+            .add("secretKey", secretKey)
+            .add("whiteRemoteAddress", whiteRemoteAddress)
+            .add("admin", admin)
+            .add("defaultTopicPerm", defaultTopicPerm)
+            .add("defaultGroupPerm", defaultGroupPerm)
+            .add("topicPerms", topicPerms)
+            .add("groupPerms", groupPerms)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/CreateTopicRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.TopicFilterType;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
@@ -139,4 +140,20 @@ public class CreateTopicRequestHeader implements CommandCustomHeader {
     public void setAttributes(String attributes) {
         this.attributes = attributes;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("topic", topic)
+            .add("defaultTopic", defaultTopic)
+            .add("readQueueNums", readQueueNums)
+            .add("writeQueueNums", writeQueueNums)
+            .add("perm", perm)
+            .add("topicFilterType", topicFilterType)
+            .add("topicSysFlag", topicSysFlag)
+            .add("order", order)
+            .add("attributes", attributes)
+            .add("force", force)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/EndTransactionRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.sysflag.MessageSysFlag;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
@@ -118,14 +119,14 @@ public class EndTransactionRequestHeader implements CommandCustomHeader {
 
     @Override
     public String toString() {
-        return "EndTransactionRequestHeader{" +
-            "producerGroup='" + producerGroup + '\'' +
-            ", tranStateTableOffset=" + tranStateTableOffset +
-            ", commitLogOffset=" + commitLogOffset +
-            ", commitOrRollback=" + commitOrRollback +
-            ", fromTransactionCheck=" + fromTransactionCheck +
-            ", msgId='" + msgId + '\'' +
-            ", transactionId='" + transactionId + '\'' +
-            '}';
+        return MoreObjects.toStringHelper(this)
+            .add("producerGroup", producerGroup)
+            .add("tranStateTableOffset", tranStateTableOffset)
+            .add("commitLogOffset", commitLogOffset)
+            .add("commitOrRollback", commitOrRollback)
+            .add("fromTransactionCheck", fromTransactionCheck)
+            .add("msgId", msgId)
+            .add("transactionId", transactionId)
+            .toString();
     }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java

@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -44,4 +45,12 @@ public class GetConsumeStatsRequestHeader implements CommandCustomHeader {
     public void setTopic(String topic) {
         this.topic = topic;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -36,4 +37,11 @@ public class GetConsumerListByGroupRequestHeader implements CommandCustomHeader
     public void setConsumerGroup(String consumerGroup) {
         this.consumerGroup = consumerGroup;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -57,4 +58,13 @@ public class GetConsumerRunningInfoRequestHeader implements CommandCustomHeader
     public void setJstackEnable(boolean jstackEnable) {
         this.jstackEnable = jstackEnable;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("clientId", clientId)
+            .add("jstackEnable", jstackEnable)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java

@@ -17,6 +17,7 @@
 
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -57,4 +58,13 @@ public class GetConsumerStatusRequestHeader implements CommandCustomHeader {
     public void setClientAddr(String clientAddr) {
         this.clientAddr = clientAddr;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("topic", topic)
+            .add("group", group)
+            .add("clientAddr", clientAddr)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -71,4 +72,13 @@ public class GetMaxOffsetRequestHeader extends TopicQueueRequestHeader {
     public void setCommitted(final boolean committed) {
         this.committed = committed;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("committed", committed)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -53,4 +54,12 @@ public class GetMinOffsetRequestHeader extends TopicQueueRequestHeader {
     public void setQueueId(Integer queueId) {
         this.queueId = queueId;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/PopMessageRequestHeader.java

@@ -16,6 +16,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -152,4 +153,21 @@ public class PopMessageRequestHeader implements CommandCustomHeader {
     public boolean isOrder() {
         return this.order != null && this.order.booleanValue();
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("maxMsgNums", maxMsgNums)
+            .add("invisibleTime", invisibleTime)
+            .add("pollTime", pollTime)
+            .add("bornTime", bornTime)
+            .add("initMode", initMode)
+            .add("expType", expType)
+            .add("exp", exp)
+            .add("order", order)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/PullMessageRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import java.util.HashMap;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
@@ -267,4 +268,22 @@ public class PullMessageRequestHeader extends TopicQueueRequestHeader implements
     public void setMaxMsgBytes(Integer maxMsgBytes) {
         this.maxMsgBytes = maxMsgBytes;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("queueOffset", queueOffset)
+            .add("maxMsgBytes", maxMsgBytes)
+            .add("maxMsgNums", maxMsgNums)
+            .add("sysFlag", sysFlag)
+            .add("commitOffset", commitOffset)
+            .add("suspendTimeoutMillis", suspendTimeoutMillis)
+            .add("subscription", subscription)
+            .add("subVersion", subVersion)
+            .add("expressionType", expressionType)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -65,4 +66,12 @@ public class SearchOffsetRequestHeader extends TopicQueueRequestHeader {
         this.timestamp = timestamp;
     }
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("timestamp", timestamp)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeader.java

@@ -20,10 +20,14 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
+import java.util.HashMap;
+import org.apache.rocketmq.common.protocol.RequestCode;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
+import org.apache.rocketmq.remoting.protocol.RemotingCommand;
 
 public class SendMessageRequestHeader extends TopicQueueRequestHeader {
     @CFNotNull
@@ -163,4 +167,119 @@ public class SendMessageRequestHeader extends TopicQueueRequestHeader {
     public void setBatch(boolean batch) {
         this.batch = batch;
     }
+
+    public static SendMessageRequestHeader parseRequestHeader(RemotingCommand request) throws RemotingCommandException {
+        SendMessageRequestHeaderV2 requestHeaderV2 = null;
+        SendMessageRequestHeader requestHeader = null;
+        switch (request.getCode()) {
+            case RequestCode.SEND_BATCH_MESSAGE:
+            case RequestCode.SEND_MESSAGE_V2:
+                requestHeaderV2 =
+                    (SendMessageRequestHeaderV2) request
+                        .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class);
+            case RequestCode.SEND_MESSAGE:
+                if (null == requestHeaderV2) {
+                    requestHeader =
+                        (SendMessageRequestHeader) request
+                            .decodeCommandCustomHeader(SendMessageRequestHeader.class);
+                } else {
+                    requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2);
+                }
+            default:
+                break;
+        }
+        return requestHeader;
+    }
+
+    public static SendMessageRequestHeaderV2 decodeSendMessageHeaderV2(RemotingCommand request)
+        throws RemotingCommandException {
+        SendMessageRequestHeaderV2 r = new SendMessageRequestHeaderV2();
+        HashMap<String, String> fields = request.getExtFields();
+        if (fields == null) {
+            throw new RemotingCommandException("the ext fields is null");
+        }
+
+        String s = fields.get("a");
+        checkNotNull(s, "the custom field <a> is null");
+        r.setA(s);
+
+        s = fields.get("b");
+        checkNotNull(s, "the custom field <b> is null");
+        r.setB(s);
+
+        s = fields.get("c");
+        checkNotNull(s, "the custom field <c> is null");
+        r.setC(s);
+
+        s = fields.get("d");
+        checkNotNull(s, "the custom field <d> is null");
+        r.setD(Integer.parseInt(s));
+
+        s = fields.get("e");
+        checkNotNull(s, "the custom field <e> is null");
+        r.setE(Integer.parseInt(s));
+
+        s = fields.get("f");
+        checkNotNull(s, "the custom field <f> is null");
+        r.setF(Integer.parseInt(s));
+
+        s = fields.get("g");
+        checkNotNull(s, "the custom field <g> is null");
+        r.setG(Long.parseLong(s));
+
+        s = fields.get("h");
+        checkNotNull(s, "the custom field <h> is null");
+        r.setH(Integer.parseInt(s));
+
+        s = fields.get("i");
+        if (s != null) {
+            r.setI(s);
+        }
+
+        s = fields.get("j");
+        if (s != null) {
+            r.setJ(Integer.parseInt(s));
+        }
+
+        s = fields.get("k");
+        if (s != null) {
+            r.setK(Boolean.parseBoolean(s));
+        }
+
+        s = fields.get("l");
+        if (s != null) {
+            r.setL(Integer.parseInt(s));
+        }
+
+        s = fields.get("m");
+        if (s != null) {
+            r.setM(Boolean.parseBoolean(s));
+        }
+        return r;
+    }
+
+    private static void checkNotNull(String s, String msg) throws RemotingCommandException {
+        if (s == null) {
+            throw new RemotingCommandException(msg);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("producerGroup", producerGroup)
+            .add("topic", topic)
+            .add("defaultTopic", defaultTopic)
+            .add("defaultTopicQueueNums", defaultTopicQueueNums)
+            .add("queueId", queueId)
+            .add("sysFlag", sysFlag)
+            .add("bornTimestamp", bornTimestamp)
+            .add("flag", flag)
+            .add("properties", properties)
+            .add("reconsumeTimes", reconsumeTimes)
+            .add("unitMode", unitMode)
+            .add("batch", batch)
+            .add("maxReconsumeTimes", maxReconsumeTimes)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java

@@ -20,6 +20,7 @@ package org.apache.rocketmq.common.protocol.header;
 import java.util.HashMap;
 
 import org.apache.rocketmq.remoting.protocol.FastCodesHeader;
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.remoting.CommandCustomHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.annotation.CFNullable;
@@ -288,4 +289,23 @@ public class SendMessageRequestHeaderV2 implements CommandCustomHeader, FastCode
     public void setM(boolean m) {
         this.m = m;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("a", a)
+            .add("b", b)
+            .add("c", c)
+            .add("d", d)
+            .add("e", e)
+            .add("f", f)
+            .add("g", g)
+            .add("h", h)
+            .add("i", i)
+            .add("j", j)
+            .add("k", k)
+            .add("l", l)
+            .add("m", m)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java

@@ -20,6 +20,7 @@
  */
 package org.apache.rocketmq.common.protocol.header;
 
+import com.google.common.base.MoreObjects;
 import org.apache.rocketmq.common.rpc.TopicQueueRequestHeader;
 import org.apache.rocketmq.remoting.annotation.CFNotNull;
 import org.apache.rocketmq.remoting.exception.RemotingCommandException;
@@ -73,4 +74,14 @@ public class UpdateConsumerOffsetRequestHeader extends TopicQueueRequestHeader {
     public void setCommitOffset(Long commitOffset) {
         this.commitOffset = commitOffset;
     }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("consumerGroup", consumerGroup)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("commitOffset", commitOffset)
+            .toString();
+    }
 }

common/src/main/java/org/apache/rocketmq/common/subscription/CustomizedRetryPolicy.java

@@ -48,6 +48,13 @@ public class CustomizedRetryPolicy implements RetryPolicy {
         TimeUnit.HOURS.toMillis(2)
     };
 
+    public CustomizedRetryPolicy() {
+    }
+
+    public CustomizedRetryPolicy(long[] next) {
+        this.next = next;
+    }
+
     public long[] getNext() {
         return next;
     }

common/src/main/java/org/apache/rocketmq/common/subscription/ExponentialRetryPolicy.java

@@ -20,14 +20,20 @@ package org.apache.rocketmq.common.subscription;
 import com.google.common.base.MoreObjects;
 import java.util.concurrent.TimeUnit;
 
-/**
- * next delay time = min(max, initial * multiplier^reconsumeTimes)
- */
 public class ExponentialRetryPolicy implements RetryPolicy {
     private long initial = TimeUnit.SECONDS.toMillis(5);
     private long max = TimeUnit.HOURS.toMillis(2);
     private long multiplier = 2;
 
+    public ExponentialRetryPolicy() {
+    }
+
+    public ExponentialRetryPolicy(long initial, long max, long multiplier) {
+        this.initial = initial;
+        this.max = max;
+        this.multiplier = multiplier;
+    }
+
     public long getInitial() {
         return initial;
     }

common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.thread;
+
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.UtilAll;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+
+public class ThreadPoolMonitor {
+    private static InternalLogger jstackLogger = InternalLoggerFactory.getLogger(ThreadPoolMonitor.class);
+    private static InternalLogger waterMarkLogger = InternalLoggerFactory.getLogger(ThreadPoolMonitor.class);
+
+    private static final List<ThreadPoolWrapper> MONITOR_EXECUTOR = new CopyOnWriteArrayList<>();
+    private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor(
+        new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build()
+    );
+
+    private static volatile long threadPoolStatusPeriodTime = TimeUnit.SECONDS.toMillis(3);
+    private static volatile boolean enablePrintJstack = true;
+    private static volatile long jstackPeriodTime = 60000;
+    private static volatile long jstackTime = System.currentTimeMillis();
+
+    public static void config(InternalLogger jstackLoggerConfig, InternalLogger waterMarkLoggerConfig,
+        boolean enablePrintJstack, long jstackPeriodTimeConfig, long threadPoolStatusPeriodTimeConfig) {
+        jstackLogger = jstackLoggerConfig;
+        waterMarkLogger = waterMarkLoggerConfig;
+        threadPoolStatusPeriodTime = threadPoolStatusPeriodTimeConfig;
+        ThreadPoolMonitor.enablePrintJstack = enablePrintJstack;
+        jstackPeriodTime = jstackPeriodTimeConfig;
+    }
+
+    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,
+        int maximumPoolSize,
+        long keepAliveTime,
+        TimeUnit unit,
+        String name,
+        int queueCapacity) {
+        return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity, Collections.emptyList());
+    }
+
+    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,
+        int maximumPoolSize,
+        long keepAliveTime,
+        TimeUnit unit,
+        String name,
+        int queueCapacity,
+        ThreadPoolStatusMonitor... threadPoolStatusMonitors) {
+        return createAndMonitor(corePoolSize, maximumPoolSize, keepAliveTime, unit, name, queueCapacity,
+            Lists.newArrayList(threadPoolStatusMonitors));
+    }
+
+    public static ThreadPoolExecutor createAndMonitor(int corePoolSize,
+        int maximumPoolSize,
+        long keepAliveTime,
+        TimeUnit unit,
+        String name,
+        int queueCapacity,
+        List<ThreadPoolStatusMonitor> threadPoolStatusMonitors) {
+        ThreadPoolExecutor executor = new ThreadPoolExecutor(
+            corePoolSize,
+            maximumPoolSize,
+            keepAliveTime,
+            unit,
+            new LinkedBlockingQueue<>(queueCapacity),
+            new ThreadFactoryBuilder().setNameFormat(name + "-%d").build(),
+            new ThreadPoolExecutor.DiscardOldestPolicy());
+        List<ThreadPoolStatusMonitor> printers = Lists.newArrayList(new ThreadPoolQueueSizeMonitor(queueCapacity));
+        printers.addAll(threadPoolStatusMonitors);
+
+        MONITOR_EXECUTOR.add(ThreadPoolWrapper.builder()
+            .name(name)
+            .threadPoolExecutor(executor)
+            .statusPrinters(printers)
+            .build());
+        return executor;
+    }
+
+    public static void logThreadPoolStatus() {
+        for (ThreadPoolWrapper threadPoolWrapper : MONITOR_EXECUTOR) {
+            List<ThreadPoolStatusMonitor> monitors = threadPoolWrapper.getStatusPrinters();
+            for (ThreadPoolStatusMonitor monitor : monitors) {
+                double value = monitor.value(threadPoolWrapper.getThreadPoolExecutor());
+                waterMarkLogger.info("\t{}\t{}\t{}", threadPoolWrapper.getName(),
+                    monitor.describe(),
+                    value);
+
+                if (enablePrintJstack) {
+                    if (monitor.needPrintJstack(threadPoolWrapper.getThreadPoolExecutor(), value) &&
+                        System.currentTimeMillis() - jstackTime > jstackPeriodTime) {
+                        jstackTime = System.currentTimeMillis();
+                        jstackLogger.warn("jstack start\n{}", UtilAll.jstack());
+                    }
+                }
+            }
+        }
+    }
+
+    public static void init() {
+        MONITOR_SCHEDULED.scheduleAtFixedRate(ThreadPoolMonitor::logThreadPoolStatus, 20,
+            threadPoolStatusPeriodTime, TimeUnit.MILLISECONDS);
+    }
+
+    public static void shutdown() {
+        MONITOR_SCHEDULED.shutdown();
+    }
+}

common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolQueueSizeMonitor.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.thread;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+public class ThreadPoolQueueSizeMonitor implements ThreadPoolStatusMonitor {
+
+    private final int maxQueueCapacity;
+
+    public ThreadPoolQueueSizeMonitor(int maxQueueCapacity) {
+        this.maxQueueCapacity = maxQueueCapacity;
+    }
+
+    @Override
+    public String describe() {
+        return "queueSize";
+    }
+
+    @Override
+    public double value(ThreadPoolExecutor executor) {
+        return executor.getQueue().size();
+    }
+
+    @Override
+    public boolean needPrintJstack(ThreadPoolExecutor executor, double value) {
+        return value > maxQueueCapacity * 0.85;
+    }
+}

common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolStatusMonitor.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.thread;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+public interface ThreadPoolStatusMonitor {
+
+    String describe();
+
+    double value(ThreadPoolExecutor executor);
+
+    boolean needPrintJstack(ThreadPoolExecutor executor, double value);
+}

common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolWrapper.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.thread;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+
+public class ThreadPoolWrapper {
+    private String name;
+    private ThreadPoolExecutor threadPoolExecutor;
+    private List<ThreadPoolStatusMonitor> statusPrinters;
+
+    ThreadPoolWrapper(final String name, final ThreadPoolExecutor threadPoolExecutor,
+        final List<ThreadPoolStatusMonitor> statusPrinters) {
+        this.name = name;
+        this.threadPoolExecutor = threadPoolExecutor;
+        this.statusPrinters = statusPrinters;
+    }
+
+    public static class ThreadPoolWrapperBuilder {
+        private String name;
+        private ThreadPoolExecutor threadPoolExecutor;
+        private List<ThreadPoolStatusMonitor> statusPrinters;
+
+        ThreadPoolWrapperBuilder() {
+        }
+
+        public ThreadPoolWrapper.ThreadPoolWrapperBuilder name(final String name) {
+            this.name = name;
+            return this;
+        }
+
+        public ThreadPoolWrapper.ThreadPoolWrapperBuilder threadPoolExecutor(
+            final ThreadPoolExecutor threadPoolExecutor) {
+            this.threadPoolExecutor = threadPoolExecutor;
+            return this;
+        }
+
+        public ThreadPoolWrapper.ThreadPoolWrapperBuilder statusPrinters(
+            final List<ThreadPoolStatusMonitor> statusPrinters) {
+            this.statusPrinters = statusPrinters;
+            return this;
+        }
+
+        public ThreadPoolWrapper build() {
+            return new ThreadPoolWrapper(this.name, this.threadPoolExecutor, this.statusPrinters);
+        }
+
+        @java.lang.Override
+        public java.lang.String toString() {
+            return "ThreadPoolWrapper.ThreadPoolWrapperBuilder(name=" + this.name + ", threadPoolExecutor=" + this.threadPoolExecutor + ", statusPrinters=" + this.statusPrinters + ")";
+        }
+    }
+
+    public static ThreadPoolWrapper.ThreadPoolWrapperBuilder builder() {
+        return new ThreadPoolWrapper.ThreadPoolWrapperBuilder();
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public ThreadPoolExecutor getThreadPoolExecutor() {
+        return this.threadPoolExecutor;
+    }
+
+    public List<ThreadPoolStatusMonitor> getStatusPrinters() {
+        return this.statusPrinters;
+    }
+
+    public void setName(final String name) {
+        this.name = name;
+    }
+
+    public void setThreadPoolExecutor(final ThreadPoolExecutor threadPoolExecutor) {
+        this.threadPoolExecutor = threadPoolExecutor;
+    }
+
+    public void setStatusPrinters(final List<ThreadPoolStatusMonitor> statusPrinters) {
+        this.statusPrinters = statusPrinters;
+    }
+
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        ThreadPoolWrapper wrapper = (ThreadPoolWrapper) o;
+        return Objects.equal(name, wrapper.name) && Objects.equal(threadPoolExecutor, wrapper.threadPoolExecutor) && Objects.equal(statusPrinters, wrapper.statusPrinters);
+    }
+
+    @Override public int hashCode() {
+        return Objects.hashCode(name, threadPoolExecutor, statusPrinters);
+    }
+
+    @Override public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("name", name)
+            .add("threadPoolExecutor", threadPoolExecutor)
+            .add("statusPrinters", statusPrinters)
+            .toString();
+    }
+}

common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.common.utils;
+
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import org.apache.commons.codec.binary.Hex;
+
+public class BinaryUtil {
+    public static byte[] calculateMd5(byte[] binaryData) {
+        MessageDigest messageDigest = null;
+        try {
+            messageDigest = MessageDigest.getInstance("MD5");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("MD5 algorithm not found.");
+        }
+        messageDigest.update(binaryData);
+        return messageDigest.digest();
+    }
+
+    public static String generateMd5(String bodyStr) {
+        byte[] bytes = calculateMd5(bodyStr.getBytes(Charset.forName("UTF-8")));
+        return Hex.encodeHexString(bytes, false);
+    }
+
+    public static String generateMd5(byte[] content) {
+        byte[] bytes = calculateMd5(content);
+        return Hex.encodeHexString(bytes, false);
+    }
+}

distribution/bin/mqproxy

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "$ROCKETMQ_HOME" ] ; then
+  ## resolve links - $0 may be a link to maven's home
+  PRG="$0"
+
+  # need this for relative symlinks
+  while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+      PRG="$link"
+    else
+      PRG="`dirname "$PRG"`/$link"
+    fi
+  done
+
+  saveddir=`pwd`
+
+  ROCKETMQ_HOME=`dirname "$PRG"`/..
+
+  # make it fully qualified
+  ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd`
+
+  cd "$saveddir"
+fi
+
+export ROCKETMQ_HOME
+
+sh ${ROCKETMQ_HOME}/bin/runserver.sh org.apache.rocketmq.proxy.ProxyStartup $@

distribution/bin/mqproxy.cmd

@@ -0,0 +1,23 @@
+@echo off
+rem Licensed to the Apache Software Foundation (ASF) under one or more
+rem contributor license agreements.  See the NOTICE file distributed with
+rem this work for additional information regarding copyright ownership.
+rem The ASF licenses this file to You under the Apache License, Version 2.0
+rem (the "License"); you may not use this file except in compliance with
+rem the License.  You may obtain a copy of the License at
+rem
+rem     http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem Unless required by applicable law or agreed to in writing, software
+rem distributed under the License is distributed on an "AS IS" BASIS,
+rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem See the License for the specific language governing permissions and
+rem limitations under the License.
+
+if not exist "%ROCKETMQ_HOME%\bin\runserver.cmd" echo Please set the ROCKETMQ_HOME variable in your environment! & EXIT /B 1
+
+call "%ROCKETMQ_HOME%\bin\runserver.cmd" org.apache.rocketmq.proxy.ProxyStartup %*
+
+IF %ERRORLEVEL% EQU 0 (
+    ECHO "Proxy starts OK"
+)

distribution/bin/mqshutdown

@@ -58,6 +58,20 @@ case $1 in
 
     echo "Send shutdown request to mqnamesrv(${pid}) OK"
     ;;
+    proxy)
+
+    pid=`ps ax | grep -i 'org.apache.rocketmq.proxy.ProxyStartup' |grep java | grep -v grep | awk '{print $1}'`
+    if [ -z "$pid" ] ; then
+            echo "No mqproxy running."
+            exit -1;
+    fi
+
+    echo "The mqproxy(${pid}) is running..."
+
+    kill ${pid}
+
+    echo "Send shutdown request to mqproxy(${pid}) OK"
+    ;;
     *)
-    echo "Useage: mqshutdown broker | namesrv"
+    echo "Useage: mqshutdown broker | namesrv | proxy"
 esac

distribution/conf/logback_proxy.xml

@@ -0,0 +1,420 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<configuration>
+
+    <appender name="RocketmqProxyAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/proxy.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/proxy.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>128MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqProxyAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqProxyAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqProxyWatermarkAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/proxy_watermark.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/proxy_watermark.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>128MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8}%m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqProxyWatermarkAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqProxyWatermarkAppender_inner"/>
+    </appender>
+
+    <!-- Below is the logger configuration for broker-->
+    <appender name="DefaultAppender"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker_default.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker_default.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <appender name="RocketmqBrokerAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/broker.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/broker.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>20</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>128MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqBrokerAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqBrokerAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqProtectionAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/protection.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/protection.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqProtectionAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqProtectionAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqWaterMarkAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/watermark.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/watermark.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqWaterMarkAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqWaterMarkAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqStoreAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/store.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/store.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>128MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqStoreAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqStoreAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqRemotingAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/remoting.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/remoting.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqRemotingAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqRemotingAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqStoreErrorAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/storeerror.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/storeerror.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqStoreErrorAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqStoreErrorAppender_inner"/>
+    </appender>
+
+
+    <appender name="RocketmqTransactionAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/transaction.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/transaction.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqTransactionAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqTransactionAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqRebalanceLockAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/lock.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/lock.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>5</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqRebalanceLockAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqRebalanceLockAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqFilterAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/filter.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/filter.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+    <appender name="RocketmqFilterAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqFilterAppender_inner"/>
+    </appender>
+
+    <appender name="RocketmqStatsAppender"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/stats.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/stats.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>5</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>100MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <appender name="RocketmqCommercialAppender"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/${brokerLogDir}/commercial.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/${brokerLogDir}/commercial.%i.log.gz</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>500MB</maxFileSize>
+        </triggeringPolicy>
+    </appender>
+
+    <appender name="RocketmqPopAppender_inner"
+              class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${user.home}/logs/rocketmqlogs/pop.log</file>
+        <append>true</append>
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>${user.home}/logs/rocketmqlogs/otherdays/pop.%i.log
+            </fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>20</maxIndex>
+        </rollingPolicy>
+        <triggeringPolicy
+            class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>128MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <appender name="RocketmqPopAppender" class="ch.qos.logback.classic.AsyncAppender">
+        <appender-ref ref="RocketmqPopAppender_inner"/>
+    </appender>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <append>true</append>
+        <encoder>
+            <pattern>%d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n</pattern>
+            <charset class="java.nio.charset.Charset">UTF-8</charset>
+        </encoder>
+    </appender>
+
+    <logger name="RocketmqBroker" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqBrokerAppender"/>
+    </logger>
+
+    <logger name="RocketmqProtection" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqProtectionAppender"/>
+    </logger>
+
+    <logger name="RocketmqWaterMark" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqWaterMarkAppender"/>
+    </logger>
+
+    <logger name="RocketmqCommon" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqBrokerAppender"/>
+    </logger>
+
+    <logger name="RocketmqStore" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqStoreAppender"/>
+    </logger>
+
+    <logger name="RocketmqStoreError" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqStoreErrorAppender"/>
+    </logger>
+
+    <logger name="RocketmqTransaction" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqTransactionAppender"/>
+    </logger>
+
+    <logger name="RocketmqRebalanceLock" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqRebalanceLockAppender"/>
+    </logger>
+
+    <logger name="RocketmqRemoting" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqRemotingAppender"/>
+    </logger>
+
+    <logger name="RocketmqStats" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqStatsAppender"/>
+    </logger>
+
+    <logger name="RocketmqCommercial" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqCommercialAppender"/>
+    </logger>
+
+    <logger name="RocketmqFilter" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="RocketmqFilterAppender"/>
+    </logger>
+
+    <logger name="RocketmqConsole" additivity="false">
+        <level value="INFO"/>
+        <appender-ref ref="STDOUT"/>
+    </logger>
+
+    <logger name="RocketmqPop" additivity="false">
+        <level value="INFO" />
+        <appender-ref ref="RocketmqPopAppender" />
+    </logger>
+
+    <logger name="RocketmqProxy" additivity="false">
+        <level value="INFO" />
+        <appender-ref ref="RocketmqProxyAppender" />
+    </logger>
+
+    <logger name="RocketmqProxyWatermark" additivity="false">
+        <level value="INFO" />
+        <appender-ref ref="RocketmqProxyWatermarkAppender" />
+    </logger>
+
+    <root>
+        <level value="INFO"/>
+        <appender-ref ref="DefaultAppender"/>
+    </root>
+</configuration>

distribution/conf/rmq-proxy.json

@@ -0,0 +1,3 @@
+{
+  
+}

distribution/pom.xml

@@ -38,6 +38,10 @@
                     <groupId>org.apache.rocketmq</groupId>
                     <artifactId>rocketmq-broker</artifactId>
                 </dependency>
+                <dependency>
+                    <groupId>org.apache.rocketmq</groupId>
+                    <artifactId>rocketmq-proxy</artifactId>
+                </dependency>
                 <dependency>
                     <groupId>org.apache.rocketmq</groupId>
                     <artifactId>rocketmq-client</artifactId>

docs/en/README.md

@@ -33,6 +33,9 @@
 
 - [Cluster Deployment](dledger/deploy_guide.md)：introduce how to deploy Dledger in cluster.
 
+- [Proxy Deployment](proxy/deploy_guide.md)
+  Introduce how to deploy proxy (both `Local` mode and `Cluster` mode).
+
 ### 5. Operation and maintenance management
 - [Operation](operation.md)：introduce RocketMQ's deployment modes that including single-master mode, multi-master mode, multi-master multi-slave mode and so on, as well as the usage of operation tool mqadmin.
 

docs/en/proxy/deploy_guide.md

@@ -0,0 +1,37 @@
+# RocketMQ Proxy Deployment Guide
+
+## Overview
+
+RocketMQ Proxy supports two deployment modes, `Local` mode and `Cluster` mode.
+
+## Configuration
+
+The configuration applies to both the `Cluster` mode and `Local` mode, whose default path is
+distribution/conf/rmq-proxy.json directory.
+
+## `Cluster` mode
+
+* Set configuration field `nameSrvAddr`.
+* Set configuration field `proxyMode` to `cluster` (case insensitive).
+
+Run the command below.
+
+```shell
+nohup sh mqproxy &
+```
+
+The command will only run `Proxy` itself. It requires `Namesrv` and `Broker` components running.
+
+## `Local` mode
+
+* Set configuration field `nameSrvAddr`.
+* Set configuration field `proxyMode` to `local` (case insensitive).
+
+Run the command below.
+
+```shell
+nohup sh mqproxy &
+```
+
+The command will not only run `Proxy`, but also run `Broker`. It requires `Namesrv` only and there's no need for
+extra `Broker`.

pom.xml

@@ -113,7 +113,7 @@
         <openmessaging.version>0.3.1-alpha</openmessaging.version>
         <log4j.version>1.2.17</log4j.version>
         <snakeyaml.version>1.30</snakeyaml.version>
-        <commons-codec.version>1.9</commons-codec.version>
+        <commons-codec.version>1.13</commons-codec.version>
         <logging-log4j.version>2.17.1</logging-log4j.version>
         <commons-validator.version>1.7</commons-validator.version>
         <zstd-jni.version>1.5.2-2</zstd-jni.version>
@@ -124,10 +124,13 @@
         <annotations-api.version>6.0.53</annotations-api.version>
         <extra-enforcer-rules.version>1.0-beta-4</extra-enforcer-rules.version>
         <concurrentlinkedhashmap-lru.version>1.4.2</concurrentlinkedhashmap-lru.version>
+        <rocketmq-proto.version>2.0.0</rocketmq-proto.version>
+        <grpc.version>1.45.0</grpc.version>
+        <protobuf-java-util.version>3.20.1</protobuf-java-util.version>
 
         <!-- Test dependencies -->
         <junit.version>4.13.2</junit.version>
-        <assertj-core.version>2.6.0</assertj-core.version>
+        <assertj-core.version>3.22.0</assertj-core.version>
         <mockito-core.version>3.10.0</mockito-core.version>
         <awaitility.version>4.1.0</awaitility.version>
         <truth.version>0.30</truth.version>
@@ -156,7 +159,6 @@
         <!-- Exclude all generated code -->
         <sonar.jacoco.itReportPath>${project.basedir}/../test/target/jacoco-it.exec</sonar.jacoco.itReportPath>
         <sonar.exclusions>file:**/generated-sources/**,**/test/**</sonar.exclusions>
-
     </properties>
 
     <modules>
@@ -176,6 +178,7 @@
         <module>acl</module>
         <module>example</module>
         <module>container</module>
+        <module>proxy</module>
     </modules>
 
     <build>
@@ -284,9 +287,11 @@
                     <excludes>
                         <exclude>.gitignore</exclude>
                         <exclude>.travis.yml</exclude>
+                        <exclude>README.md</exclude>
                         <exclude>CONTRIBUTING.md</exclude>
                         <exclude>bin/README.md</exclude>
                         <exclude>.github/**</exclude>
+                        <exclude>src/test/resources/**</exclude>
                         <exclude>src/test/resources/certs/*</exclude>
                         <exclude>src/test/**/*.log</exclude>
                         <exclude>src/test/resources/META-INF/service/*</exclude>
@@ -295,6 +300,7 @@
                         <exclude>*/*.iml</exclude>
                         <exclude>docs/**</exclude>
                         <exclude>localbin/**</exclude>
+                        <exclude>conf/rmq-proxy.json</exclude>
                     </excludes>
                 </configuration>
             </plugin>
@@ -476,6 +482,11 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>${project.groupId}</groupId>
+                <artifactId>rocketmq-proto</artifactId>
+                <version>${rocketmq-proto.version}</version>
+            </dependency>
             <dependency>
                 <groupId>${project.groupId}</groupId>
                 <artifactId>rocketmq-client</artifactId>
@@ -551,6 +562,11 @@
                 <artifactId>rocketmq-example</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>${project.groupId}</groupId>
+                <artifactId>rocketmq-proxy</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-api</artifactId>
@@ -712,13 +728,43 @@
                 <groupId>org.awaitility</groupId>
                 <artifactId>awaitility</artifactId>
                 <version>${awaitility.version}</version>
-                <scope>test</scope>
             </dependency>
             <dependency>
                 <groupId>com.google.truth</groupId>
                 <artifactId>truth</artifactId>
                 <version>${truth.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-netty-shaded</artifactId>
+                <version>${grpc.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-protobuf</artifactId>
+                <version>${grpc.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-stub</artifactId>
+                <version>${grpc.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-services</artifactId>
+                <version>${grpc.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.grpc</groupId>
+                <artifactId>grpc-testing</artifactId>
+                <version>${grpc.version}</version>
+                <scope>test</scope>
+            </dependency>
+            <dependency>
+                <groupId>com.google.protobuf</groupId>
+                <artifactId>protobuf-java-util</artifactId>
+                <version>${protobuf-java-util.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -727,16 +773,19 @@
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <version>${junit.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.assertj</groupId>
             <artifactId>assertj-core</artifactId>
             <version>${assertj-core.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
             <version>${mockito-core.version}</version>
+            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.awaitility</groupId>

proxy/README.md

@@ -0,0 +1,60 @@
+rocketmq-proxy
+--------
+
+## Introduction
+
+`RocketMQ Proxy` is a stateless component that makes full use of the newly introduced `pop` consumption mechanism to
+achieve stateless consumption behavior. `gRPC` protocol is supported by `Proxy` now and all the message types
+including `normal`, `fifo`, `transaction` and `delay` are supported via `pop` consumption mode. `Proxy` will translate
+incoming traffic into customized `Remoting` protocol to access `Broker` and `Namesrv`.
+
+`Proxy` also handles SSL, authorization/authentication and logging/tracing/metrics and is in charge of connection
+management and traffic governance.
+
+### Multi-language support.
+
+`gRPC` combined with `Protocol Buffer` makes it easy to implement clients with both `java` and other programming
+languages while the server side doesn't need extra work to support different programming languages.
+See [rocketmq-clients](https://github.com/apache/rocketmq-clients) for more information.
+
+### Multi-protocol support.
+
+With `Proxy` served as a traffic interface, it's convenient to implement multiple protocols upon proxy. `gRPC` protocol
+is implemented first and the customized `Remoting` protocol will be implemented later. HTTP/1.1 will also be taken into
+consideration.
+
+## Architecture
+
+`RocketMQ Proxy` has two deployment modes: `Cluster` mode and `Local` mode. With both modes, `Pop` mode is natively
+supported in `Proxy`.
+
+### `Cluster` mode
+
+While in `Cluster` mode, `Proxy` is an independent cluster that communicates with `Broker` with remote procedure call.
+In this scenario, `Proxy` acts as a stateless computing component while `Broker` is a stateful component with local
+storage. This form of deployment introduces the architecture of separation of computing and storage for RocketMQ.
+
+Due to the separation of computing and storage, `RocketMQ Proxy` can be scaled out indefinitely in `Cluster` mode to
+handle traffic peak while `Broker` can focus on storage engine and high availability.
+
+![](../docs/en/images/rocketmq_proxy_cluster_mode.png)
+
+### `Local` mode
+
+`Proxy` in `Local` mode has more similarity with `RocketMQ` 4.x version, which is easily deployed or upgraded for
+current RocketMQ users. With `Local` mode, `Proxy` deployed with `Broker` in the same process with inter-process
+communication so the network overhead is reduced compared to `Cluster` mode.
+
+![](../docs/en/images/rocketmq_proxy_local_mode.png)
+
+## Deploy guide
+
+See [Proxy Deployment](../docs/en/proxy/deploy_guide.md)
+
+## Related
+
+* [rocketmq-apis](https://github.com/apache/rocketmq-apis): Common communication protocol between server and client.
+* [rocketmq-clients](https://github.com/apache/rocketmq-clients): Collection of Polyglot Clients for Apache RocketMQ.
+* [RIP-37: New and Unified APIs](https://shimo.im/docs/m5kv92OeRRU8olqX): RocketMQ proposal of new and unified APIs
+  crossing different languages.
+* [RIP-39: Support gRPC protocol](https://shimo.im/docs/gXqmeEPYgdUw5bqo): RocketMQ proposal of gRPC protocol support.

proxy/pom.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>rocketmq-all</artifactId>
+        <groupId>org.apache.rocketmq</groupId>
+        <version>5.0.0-SNAPSHOT</version>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+    <artifactId>rocketmq-proxy</artifactId>
+    <name>rocketmq-proxy ${project.version}</name>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-proto</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-broker</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.rocketmq</groupId>
+            <artifactId>rocketmq-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-netty-shaded</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-services</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java-util</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

proxy/src/main/java/org/apache/rocketmq/proxy/ProxyMode.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy;
+
+public enum ProxyMode {
+    LOCAL("LOCAL"),
+    CLUSTER("CLUSTER");
+
+    private final String mode;
+
+    ProxyMode(String mode) {
+        this.mode = mode;
+    }
+
+    public static boolean isClusterMode(String mode) {
+        if (mode == null) {
+            return false;
+        }
+        return CLUSTER.mode.equals(mode.toUpperCase());
+    }
+
+    public static boolean isClusterMode(ProxyMode mode) {
+        if (mode == null) {
+            return false;
+        }
+        return CLUSTER.equals(mode);
+    }
+
+    public static boolean isLocalMode(String mode) {
+        if (mode == null) {
+            return false;
+        }
+        return LOCAL.mode.equals(mode.toUpperCase());
+    }
+
+    public static boolean isLocalMode(ProxyMode mode) {
+        if (mode == null) {
+            return false;
+        }
+        return LOCAL.equals(mode);
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java

@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
+import java.util.Date;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.broker.BrokerController;
+import org.apache.rocketmq.broker.BrokerStartup;
+import org.apache.rocketmq.client.log.ClientLogger;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.thread.ThreadPoolMonitor;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.common.AbstractStartAndShutdown;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.grpc.GrpcServer;
+import org.apache.rocketmq.proxy.grpc.GrpcServerBuilder;
+import org.apache.rocketmq.proxy.grpc.v2.GrpcMessagingApplication;
+import org.apache.rocketmq.proxy.processor.DefaultMessagingProcessor;
+import org.apache.rocketmq.proxy.processor.MessagingProcessor;
+import org.slf4j.LoggerFactory;
+
+public class ProxyStartup {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private static final ProxyStartAndShutdown PROXY_START_AND_SHUTDOWN = new ProxyStartAndShutdown();
+
+    private static class ProxyStartAndShutdown extends AbstractStartAndShutdown {
+        @Override
+        public void appendStartAndShutdown(StartAndShutdown startAndShutdown) {
+            super.appendStartAndShutdown(startAndShutdown);
+        }
+    }
+
+    public static void main(String[] args) {
+        try {
+            ConfigurationManager.initEnv();
+            initLogger();
+            ConfigurationManager.intConfig();
+
+            // init thread pool monitor for proxy.
+            initThreadPoolMonitor();
+
+            ThreadPoolExecutor executor = createServerExecutor();
+
+            MessagingProcessor messagingProcessor = createMessagingProcessor();
+
+            // create grpcServer
+            GrpcServer grpcServer = GrpcServerBuilder.newBuilder(executor, ConfigurationManager.getProxyConfig().getGrpcServerPort())
+                .addService(createServiceProcessor(messagingProcessor))
+                .configInterceptor()
+                .build();
+            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer);
+
+            // start servers one by one.
+            PROXY_START_AND_SHUTDOWN.start();
+
+            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+                log.info("try to shutdown server");
+                try {
+                    PROXY_START_AND_SHUTDOWN.shutdown();
+                } catch (Exception e) {
+                    log.error("err when shutdown rocketmq-proxy", e);
+                }
+            }));
+        } catch (Exception e) {
+            System.err.println("find an unexpect err." + e);
+            e.printStackTrace();
+            log.error("find an unexpect err.", e);
+            System.exit(1);
+        }
+
+        System.out.printf("%s%n", new Date() + " rocketmq-proxy startup successfully");
+        log.info(new Date() + " rocketmq-proxy startup successfully");
+    }
+
+    private static MessagingProcessor createMessagingProcessor() {
+        String proxyModeStr = ConfigurationManager.getProxyConfig().getProxyMode();
+        MessagingProcessor messagingProcessor;
+
+        if (ProxyMode.isClusterMode(proxyModeStr)) {
+            messagingProcessor = DefaultMessagingProcessor.createForClusterMode();
+        } else if (ProxyMode.isLocalMode(proxyModeStr)) {
+            BrokerController brokerController = createBrokerController();
+            StartAndShutdown brokerControllerWrapper = new StartAndShutdown() {
+                @Override
+                public void start() throws Exception {
+                    brokerController.start();
+                }
+
+                @Override
+                public void shutdown() throws Exception {
+                    brokerController.shutdown();
+                }
+            };
+            PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(brokerControllerWrapper);
+            messagingProcessor = DefaultMessagingProcessor.createForLocalMode(brokerController);
+        } else {
+            throw new IllegalArgumentException("try to start grpc server with wrong mode, use 'local' or 'cluster'");
+        }
+        PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(messagingProcessor);
+        return messagingProcessor;
+    }
+
+    private static GrpcMessagingApplication createServiceProcessor(MessagingProcessor messagingProcessor) {
+        GrpcMessagingApplication application = GrpcMessagingApplication.create(messagingProcessor);
+        PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(application);
+        return application;
+    }
+
+    private static BrokerController createBrokerController() {
+        String[] brokerStartupArgs = new String[] {"-c", ConfigurationManager.getProxyConfig().getBrokerConfigPath()};
+        return BrokerStartup.createBrokerController(brokerStartupArgs);
+    }
+
+    public static ThreadPoolExecutor createServerExecutor() {
+        ProxyConfig config = ConfigurationManager.getProxyConfig();
+        int threadPoolNums = config.getGrpcThreadPoolNums();
+        int threadPoolQueueCapacity = config.getGrpcThreadPoolQueueCapacity();
+        ThreadPoolExecutor executor = ThreadPoolMonitor.createAndMonitor(
+            threadPoolNums,
+            threadPoolNums,
+            1, TimeUnit.MINUTES,
+            "GrpcRequestExecutorThread",
+            threadPoolQueueCapacity
+        );
+        PROXY_START_AND_SHUTDOWN.appendShutdown(executor::shutdown);
+        return executor;
+    }
+
+    public static void initThreadPoolMonitor() {
+        ProxyConfig config = ConfigurationManager.getProxyConfig();
+        ThreadPoolMonitor.config(
+            InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME),
+            InternalLoggerFactory.getLogger(LoggerName.PROXY_WATER_MARK_LOGGER_NAME),
+            config.isEnablePrintJstack(), config.getPrintJstackInMillis(),
+            config.getPrintThreadPoolStatusInMillis());
+        ThreadPoolMonitor.init();
+    }
+
+    public static void initLogger() throws JoranException {
+        System.setProperty("brokerLogDir", "");
+        System.setProperty(ClientLogger.CLIENT_LOG_USESLF4J, "true");
+
+        LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory();
+        JoranConfigurator configurator = new JoranConfigurator();
+        configurator.setContext(lc);
+        lc.reset();
+        //https://logback.qos.ch/manual/configuration.html
+        lc.setPackagingDataEnabled(false);
+        configurator.doConfigure(ConfigurationManager.getProxyHome() + "/conf/logback_proxy.xml");
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractCacheLoader.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+import com.google.common.cache.CacheLoader;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListenableFutureTask;
+import java.util.concurrent.ThreadPoolExecutor;
+import javax.annotation.Nonnull;
+
+public abstract class AbstractCacheLoader<K, V> extends CacheLoader<K, V> {
+    private final ThreadPoolExecutor cacheRefreshExecutor;
+
+    public AbstractCacheLoader(ThreadPoolExecutor cacheRefreshExecutor) {
+        this.cacheRefreshExecutor = cacheRefreshExecutor;
+    }
+
+    @Override
+    public ListenableFuture<V> reload(@Nonnull K key, @Nonnull V oldValue) throws Exception {
+        ListenableFutureTask<V> task = ListenableFutureTask.create(() -> {
+            try {
+                return getDirectly(key);
+            } catch (Exception e) {
+                onErr(key, e);
+                return oldValue;
+            }
+        });
+        cacheRefreshExecutor.execute(task);
+        return task;
+    }
+
+    @Override
+    public V load(@Nonnull K key) throws Exception {
+        return getDirectly(key);
+    }
+
+    protected abstract V getDirectly(K key) throws Exception;
+
+    protected abstract void onErr(K key, Exception e);
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/AbstractStartAndShutdown.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+public abstract class AbstractStartAndShutdown implements StartAndShutdown {
+
+    protected List<StartAndShutdown> startAndShutdownList = new CopyOnWriteArrayList<>();
+
+    protected void appendStartAndShutdown(StartAndShutdown startAndShutdown) {
+        this.startAndShutdownList.add(startAndShutdown);
+    }
+
+    @Override
+    public void start() throws Exception {
+        for (StartAndShutdown startAndShutdown : startAndShutdownList) {
+            startAndShutdown.start();
+        }
+    }
+
+    @Override
+    public void shutdown() throws Exception {
+        int index = startAndShutdownList.size() - 1;
+        for (; index >= 0; index--) {
+            startAndShutdownList.get(index).shutdown();
+        }
+    }
+
+    public void appendStart(Start start) {
+        this.appendStartAndShutdown(new StartAndShutdown() {
+            @Override
+            public void shutdown() throws Exception {
+
+            }
+
+            @Override
+            public void start() throws Exception {
+                start.start();
+            }
+        });
+    }
+
+    public void appendShutdown(Shutdown shutdown) {
+        this.appendStartAndShutdown(new StartAndShutdown() {
+            @Override
+            public void shutdown() throws Exception {
+                shutdown.shutdown();
+            }
+
+            @Override
+            public void start() throws Exception {
+
+            }
+        });
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/Address.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+import com.google.common.net.HostAndPort;
+import java.util.Objects;
+
+public class Address {
+
+    public enum AddressScheme {
+        IPv4,
+        IPv6,
+        DOMAIN_NAME,
+        UNRECOGNIZED
+    }
+
+    private AddressScheme addressScheme;
+    private HostAndPort hostAndPort;
+
+    public Address(AddressScheme addressScheme, HostAndPort hostAndPort) {
+        this.addressScheme = addressScheme;
+        this.hostAndPort = hostAndPort;
+    }
+
+    public AddressScheme getAddressScheme() {
+        return addressScheme;
+    }
+
+    public void setAddressScheme(AddressScheme addressScheme) {
+        this.addressScheme = addressScheme;
+    }
+
+    public HostAndPort getHostAndPort() {
+        return hostAndPort;
+    }
+
+    public void setHostAndPort(HostAndPort hostAndPort) {
+        this.hostAndPort = hostAndPort;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        Address address = (Address) o;
+        return addressScheme == address.addressScheme && Objects.equals(hostAndPort, address.hostAndPort);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(addressScheme, hostAndPort);
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/ContextVariable.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+public class ContextVariable {
+    public static final String REMOTE_ADDRESS = "remote-address";
+    public static final String LOCAL_ADDRESS = "local-address";
+    public static final String CLIENT_ID = "client-id";
+    public static final String LANGUAGE = "language";
+    public static final String CLIENT_VERSION = "client-version";
+    public static final String REMAINING_MS = "remaining-ms";
+    public static final String ACTION = "action";
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/MessageReceiptHandle.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Objects;
+
+public class MessageReceiptHandle {
+    private final String group;
+    private final String topic;
+    private final int queueId;
+    private final String messageId;
+    private final long queueOffset;
+    private final String originalReceiptHandle;
+    private final long timestamp;
+    private final int reconsumeTimes;
+    private final long expectInvisibleTime;
+
+    private String receiptHandle;
+
+    public MessageReceiptHandle(String group, String topic, int queueId, String receiptHandle, String messageId,
+        long queueOffset, int reconsumeTimes, long expectInvisibleTime) {
+        this.group = group;
+        this.topic = topic;
+        this.queueId = queueId;
+        this.receiptHandle = receiptHandle;
+        this.originalReceiptHandle = receiptHandle;
+        this.messageId = messageId;
+        this.queueOffset = queueOffset;
+        this.reconsumeTimes = reconsumeTimes;
+        this.expectInvisibleTime = expectInvisibleTime;
+        this.timestamp = System.currentTimeMillis();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        MessageReceiptHandle handle = (MessageReceiptHandle) o;
+        return queueId == handle.queueId && queueOffset == handle.queueOffset && timestamp == handle.timestamp
+            && reconsumeTimes == handle.reconsumeTimes && expectInvisibleTime == handle.expectInvisibleTime
+            && Objects.equal(group, handle.group) && Objects.equal(topic, handle.topic)
+            && Objects.equal(messageId, handle.messageId) && Objects.equal(originalReceiptHandle, handle.originalReceiptHandle)
+            && Objects.equal(receiptHandle, handle.receiptHandle);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(group, topic, queueId, messageId, queueOffset, originalReceiptHandle, timestamp,
+            reconsumeTimes, expectInvisibleTime, receiptHandle);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(this)
+            .add("group", group)
+            .add("topic", topic)
+            .add("queueId", queueId)
+            .add("messageId", messageId)
+            .add("queueOffset", queueOffset)
+            .add("originalReceiptHandle", originalReceiptHandle)
+            .add("timestamp", timestamp)
+            .add("reconsumeTimes", reconsumeTimes)
+            .add("expectInvisibleTime", expectInvisibleTime)
+            .add("receiptHandle", receiptHandle)
+            .toString();
+    }
+
+    public String getGroup() {
+        return group;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public int getQueueId() {
+        return queueId;
+    }
+
+    public String getReceiptHandle() {
+        return receiptHandle;
+    }
+
+    public String getOriginalReceiptHandle() {
+        return originalReceiptHandle;
+    }
+
+    public String getMessageId() {
+        return messageId;
+    }
+
+    public long getQueueOffset() {
+        return queueOffset;
+    }
+
+    public int getReconsumeTimes() {
+        return reconsumeTimes;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public long getExpectInvisibleTime() {
+        return expectInvisibleTime;
+    }
+
+    public void update(String receiptHandle) {
+        this.receiptHandle = receiptHandle;
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyContext.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ProxyContext {
+    public static final String INNER_ACTION_PREFIX = "Inner";
+    private final Map<String, Object> value = new HashMap<>();
+
+    public static ProxyContext create() {
+        return new ProxyContext();
+    }
+
+    public static ProxyContext createForInner(String actionName) {
+        return create().setAction(INNER_ACTION_PREFIX + actionName);
+    }
+
+    public static ProxyContext createForInner(Class<?> clazz) {
+        return createForInner(clazz.getSimpleName());
+    }
+
+    public Map<String, Object> getValue() {
+        return this.value;
+    }
+
+    public ProxyContext withVal(String key, Object val) {
+        this.value.put(key, val);
+        return this;
+    }
+
+    public <T> T getVal(String key) {
+        return (T) this.value.get(key);
+    }
+
+    public ProxyContext setLocalAddress(String localAddress) {
+        this.withVal(ContextVariable.LOCAL_ADDRESS, localAddress);
+        return this;
+    }
+
+    public String getLocalAddress() {
+        return this.getVal(ContextVariable.LOCAL_ADDRESS);
+    }
+
+    public ProxyContext setRemoteAddress(String remoteAddress) {
+        this.withVal(ContextVariable.REMOTE_ADDRESS, remoteAddress);
+        return this;
+    }
+
+    public String getRemoteAddress() {
+        return this.getVal(ContextVariable.REMOTE_ADDRESS);
+    }
+
+    public ProxyContext setClientID(String clientID) {
+        this.withVal(ContextVariable.CLIENT_ID, clientID);
+        return this;
+    }
+
+    public String getClientID() {
+        return this.getVal(ContextVariable.CLIENT_ID);
+    }
+
+    public ProxyContext setLanguage(String language) {
+        this.withVal(ContextVariable.LANGUAGE, language);
+        return this;
+    }
+
+    public String getLanguage() {
+        return this.getVal(ContextVariable.LANGUAGE);
+    }
+
+    public ProxyContext setClientVersion(String clientVersion) {
+        this.withVal(ContextVariable.CLIENT_VERSION, clientVersion);
+        return this;
+    }
+
+    public String getClientVersion() {
+        return this.getVal(ContextVariable.CLIENT_VERSION);
+    }
+
+    public ProxyContext setRemainingMs(Long remainingMs) {
+        this.withVal(ContextVariable.REMAINING_MS, remainingMs);
+        return this;
+    }
+
+    public Long getRemainingMs() {
+        return this.getVal(ContextVariable.REMAINING_MS);
+    }
+
+    public ProxyContext setAction(String action) {
+        this.withVal(ContextVariable.ACTION, action);
+        return this;
+    }
+
+    public String getAction() {
+        return this.getVal(ContextVariable.ACTION);
+    }
+
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyException.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+public class ProxyException extends RuntimeException {
+
+    private final ProxyExceptionCode code;
+
+    public ProxyException(ProxyExceptionCode code, String message) {
+        super(message);
+        this.code = code;
+    }
+
+    public ProxyException(ProxyExceptionCode code, String message, Throwable cause) {
+        super(message, cause);
+        this.code = code;
+    }
+
+    public ProxyExceptionCode getCode() {
+        return code;
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/ProxyExceptionCode.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common;
+
+public enum ProxyExceptionCode {
+    INVALID_BROKER_NAME,
+    TRANSACTION_DATA_NOT_FOUND,
+    FORBIDDEN,
+    MESSAGE_PROPERTY_CONFLICT_WITH_TYPE,
+    INVALID_RECEIPT_HANDLE,
+    INTERNAL_SERVER_ERROR,
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/ReceiptHandleGroup.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class ReceiptHandleGroup {
+    private final Map<String /* msgID */, Map<String /* original handle */, MessageReceiptHandle>> receiptHandleMap = new ConcurrentHashMap<>();
+
+    public void put(String msgID, String handle, MessageReceiptHandle value) {
+        Map<String, MessageReceiptHandle> handleMap = receiptHandleMap.computeIfAbsent(msgID, msgIDKey -> new ConcurrentHashMap<>());
+        handleMap.put(handle, value);
+    }
+
+    public boolean isEmpty() {
+        return this.receiptHandleMap.isEmpty();
+    }
+
+    public MessageReceiptHandle remove(String msgID, String handle) {
+        AtomicReference<MessageReceiptHandle> resRef = new AtomicReference<>();
+        receiptHandleMap.computeIfPresent(msgID, (msgIDKey, handleMap) -> {
+            resRef.set(handleMap.remove(handle));
+            if (handleMap.isEmpty()) {
+                return null;
+            }
+            return handleMap;
+        });
+        return resRef.get();
+    }
+
+    public MessageReceiptHandle removeOne(String msgID) {
+        AtomicReference<MessageReceiptHandle> resRef = new AtomicReference<>();
+        receiptHandleMap.computeIfPresent(msgID, (msgIDKey, handleMap) -> {
+            if (handleMap.isEmpty()) {
+                return null;
+            }
+            Optional<String> handleKey = handleMap.keySet().stream().findAny();
+            resRef.set(handleMap.remove(handleKey.get()));
+            if (handleMap.isEmpty()) {
+                return null;
+            }
+            return handleMap;
+        });
+        return resRef.get();
+    }
+
+    public interface DataScanner {
+        void onData(String msgID, String handle, MessageReceiptHandle receiptHandle);
+    }
+
+    public void scan(DataScanner scanner) {
+        this.receiptHandleMap.forEach((msgID, handleMap) -> {
+            handleMap.forEach((handleStr, v) -> {
+                scanner.onData(msgID, handleStr, v);
+            });
+        });
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/Shutdown.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+public interface Shutdown {
+    void shutdown() throws Exception;
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/Start.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+public interface Start {
+    void start() throws Exception;
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/StartAndShutdown.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common;
+
+public interface StartAndShutdown extends Start, Shutdown {
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ExceptionUtils.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common.utils;
+
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+
+public class ExceptionUtils {
+
+    public static Throwable getRealException(Throwable throwable) {
+        if (throwable instanceof CompletionException || throwable instanceof ExecutionException) {
+            if (throwable.getCause() != null) {
+                throwable = throwable.getCause();
+            }
+        }
+        return throwable;
+    }
+
+    public static String getErrorDetailMessage(Throwable t) {
+        if (t == null) {
+            return null;
+        }
+        StringBuilder sb = new StringBuilder();
+        sb.append(t.getMessage()).append(". ").append(t.getClass().getSimpleName());
+
+        if (t.getStackTrace().length > 0) {
+            sb.append(". ").append(t.getStackTrace()[0]);
+        }
+        return sb.toString();
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FilterUtils.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common.utils;
+
+import java.util.Set;
+import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData;
+
+public class FilterUtils {
+    /**
+     * Whether the message's tag matches consumerGroup's SubscriptionData
+     *
+     * @param tagsSet, tagSet in {@link SubscriptionData}, tagSet empty means SubscriptionData.SUB_ALL(*)
+     * @param tags,    message's tags, null means not tag attached to the message.
+     */
+    public static boolean isTagMatched(Set<String> tagsSet, String tags) {
+        if (tagsSet.isEmpty()) {
+            return true;
+        }
+
+        if (tags == null) {
+            return false;
+        }
+
+        return tagsSet.contains(tags);
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/FutureUtils.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.common.utils;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+public class FutureUtils {
+
+    public static <T> CompletableFuture<T> appendNextFuture(CompletableFuture<T> future,
+        CompletableFuture<T> nextFuture, ExecutorService executor) {
+        future.whenCompleteAsync((t, throwable) -> {
+            if (throwable != null) {
+                nextFuture.completeExceptionally(throwable);
+            } else {
+                nextFuture.complete(t);
+            }
+        }, executor);
+        return nextFuture;
+    }
+
+    public static <T> CompletableFuture<T> addExecutor(CompletableFuture<T> future, ExecutorService executor) {
+        return appendNextFuture(future, new CompletableFuture<>(), executor);
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/common/utils/ProxyUtils.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.common.utils;
+
+public class ProxyUtils {
+
+    public static final int MAX_MSG_NUMS_FOR_POP_REQUEST = 32;
+
+    public static final String BROKER_ADDR = "brokerAddr";
+}

proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigFile.java

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.config;
+
+public interface ConfigFile {
+
+    void initData();
+}

proxy/src/main/java/org/apache/rocketmq/proxy/config/Configuration.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.config;
+
+import com.alibaba.fastjson.JSON;
+import java.io.File;
+import java.nio.file.Files;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Configuration {
+    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    private final AtomicReference<ProxyConfig> proxyConfigReference = new AtomicReference<>();
+
+    public void init() throws Exception {
+        String proxyConfigData = loadJsonConfig(ProxyConfig.CONFIG_FILE_NAME);
+        if (null == proxyConfigData) {
+            throw new RuntimeException(String.format("load configuration from file: %s error.", ProxyConfig.CONFIG_FILE_NAME));
+        }
+
+        ProxyConfig proxyConfig = JSON.parseObject(proxyConfigData, ProxyConfig.class);
+        proxyConfig.initData();
+        setProxyConfig(proxyConfig);
+    }
+
+    public static String loadJsonConfig(String configFileName) throws Exception {
+        String filePath = new File(ConfigurationManager.getProxyHome() + File.separator + "conf", configFileName).toString();
+
+        File file = new File(filePath);
+        if (!file.exists()) {
+            log.warn("the config file {} not exist", filePath);
+            return null;
+        }
+        long fileLength = file.length();
+        if (fileLength <= 0) {
+            log.warn("the config file {} length is zero", filePath);
+            return null;
+        }
+
+        return new String(Files.readAllBytes(file.toPath()));
+    }
+
+    public ProxyConfig getProxyConfig() {
+        return proxyConfigReference.get();
+    }
+
+    public void setProxyConfig(ProxyConfig proxyConfig) {
+        proxyConfigReference.set(proxyConfig);
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/config/ConfigurationManager.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.config;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.common.MixAll;
+
+public class ConfigurationManager {
+    public static final String RMQ_PROXY_HOME = "RMQ_PROXY_HOME";
+    protected static final String DEFAULT_RMQ_PROXY_HOME = System.getenv(MixAll.ROCKETMQ_HOME_ENV);
+    protected static String proxyHome;
+    protected static Configuration configuration;
+
+    public static void initEnv() {
+        proxyHome = System.getenv(RMQ_PROXY_HOME);
+        if (StringUtils.isEmpty(proxyHome)) {
+            proxyHome = System.getProperty(RMQ_PROXY_HOME, DEFAULT_RMQ_PROXY_HOME);
+        }
+    }
+
+    public static void intConfig() throws Exception {
+        configuration = new Configuration();
+        configuration.init();
+    }
+
+    public static String getProxyHome() {
+        return proxyHome;
+    }
+
+    public static ProxyConfig getProxyConfig() {
+        return configuration.getProxyConfig();
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/config/MetricCollectorMode.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.config;
+
+public enum MetricCollectorMode {
+    /**
+     * Do not collect the metric from clients.
+     */
+    OFF(0),
+    /**
+     * Collect the metric from clients to the given address.
+     */
+    ON(1),
+    /**
+     * Collect the metric by the proxy itself.
+     */
+    PROXY(2);
+    private final int ordinal;
+
+    MetricCollectorMode(int ordinal) {
+        this.ordinal = ordinal;
+    }
+
+    public int getOrdinal() {
+        return ordinal;
+    }
+
+    public static MetricCollectorMode getEnumByOrdinal(int ordinal) {
+        for (MetricCollectorMode mode : MetricCollectorMode.values()) {
+            if (mode.ordinal == ordinal) {
+                return mode;
+            }
+        }
+        return OFF;
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java

@@ -0,0 +1,858 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.config;
+
+import java.time.Duration;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.proxy.ProxyMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProxyConfig implements ConfigFile {
+    private final static Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    public final static String CONFIG_FILE_NAME = "rmq-proxy.json";
+    private static final int PROCESSOR_NUMBER = Runtime.getRuntime().availableProcessors();
+
+    private String rocketMQClusterName = "";
+
+    /**
+     * configuration for ThreadPoolMonitor
+     */
+    private boolean enablePrintJstack = true;
+    private long printJstackInMillis = Duration.ofSeconds(60).toMillis();
+    private long printThreadPoolStatusInMillis = Duration.ofSeconds(3).toMillis();
+
+    private String nameSrvAddr = "";
+    private String nameSrvDomain = "";
+    private String nameSrvDomainSubgroup = "";
+    /**
+     * gRPC
+     */
+    private String proxyMode = ProxyMode.CLUSTER.name();
+    private Integer grpcServerPort = 8081;
+    private boolean grpcTlsTestModeEnable = true;
+    private String grpcTlsKeyPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.key";
+    private String grpcTlsCertPath = ConfigurationManager.getProxyHome() + "/conf/tls/rocketmq.crt";
+    private int grpcBossLoopNum = 1;
+    private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2;
+    private boolean enableGrpcEpoll = false;
+    private int grpcThreadPoolNums = 16 + PROCESSOR_NUMBER * 2;
+    private int grpcThreadPoolQueueCapacity = 100000;
+    private String brokerConfigPath = ConfigurationManager.getProxyHome() + "/conf/broker.conf";
+    /**
+     * gRPC max message size
+     * 130M = 4M * 32 messages + 2M attributes
+     */
+    private int grpcMaxInboundMessageSize = 130 * 1024 * 1024;
+    /**
+     * max message body size, 0 or negative number means no limit for proxy
+     */
+    private int maxMessageSize = 4 * 1024 * 1024;
+    /**
+     * max user property size, 0 or negative number means no limit for proxy
+     */
+    private int maxUserPropertySize = 16 * 1024;
+    private int userPropertyMaxNum = 128;
+    /**
+     * max message group size, 0 or negative number means no limit for proxy
+     */
+    private int maxMessageGroupSize = 64;
+    private long minInvisibleTimeMillsForRecv = Duration.ofSeconds(10).toMillis();
+    private long maxInvisibleTimeMills = Duration.ofHours(12).toMillis();
+    private long maxDelayTimeMills = Duration.ofDays(1).toMillis();
+    private long maxTransactionRecoverySecond = Duration.ofHours(1).getSeconds();
+    private boolean enableTopicMessageTypeCheck = true;
+
+    private int grpcClientProducerMaxAttempts = 3;
+    private long grpcClientProducerBackoffInitialMillis = 10;
+    private long grpcClientProducerBackoffMaxMillis = 1000;
+    private int grpcClientProducerBackoffMultiplier = 2;
+    private long grpcClientConsumerLongPollingTimeoutMillis = Duration.ofSeconds(30).toMillis();
+    private int grpcClientConsumerLongPollingBatchSize = 32;
+
+    private int channelExpiredInSeconds = 60;
+    private int contextExpiredInSeconds = 30;
+
+    private int rocketmqMQClientNum = 6;
+
+    private long grpcProxyRelayRequestTimeoutInSeconds = 5;
+    private int grpcProducerThreadPoolNums = PROCESSOR_NUMBER;
+    private int grpcProducerThreadQueueCapacity = 10000;
+    private int grpcConsumerThreadPoolNums = PROCESSOR_NUMBER;
+    private int grpcConsumerThreadQueueCapacity = 10000;
+    private int grpcRouteThreadPoolNums = PROCESSOR_NUMBER;
+    private int grpcRouteThreadQueueCapacity = 10000;
+    private int grpcClientManagerThreadPoolNums = PROCESSOR_NUMBER;
+    private int grpcClientManagerThreadQueueCapacity = 10000;
+    private int grpcTransactionThreadPoolNums = PROCESSOR_NUMBER;
+    private int grpcTransactionThreadQueueCapacity = 10000;
+
+    private int producerProcessorThreadPoolNums = PROCESSOR_NUMBER;
+    private int producerProcessorThreadPoolQueueCapacity = 10000;
+    private int consumerProcessorThreadPoolNums = PROCESSOR_NUMBER;
+    private int consumerProcessorThreadPoolQueueCapacity = 10000;
+
+    private int topicRouteServiceCacheExpiredInSeconds = 20;
+    private int topicRouteServiceCacheMaxNum = 20000;
+    private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER;
+    private int topicRouteServiceThreadPoolQueueCapacity = 5000;
+
+    private int topicConfigCacheExpiredInSeconds = 20;
+    private int topicConfigCacheMaxNum = 20000;
+    private int subscriptionGroupConfigCacheExpiredInSeconds = 20;
+    private int subscriptionGroupConfigCacheMaxNum = 20000;
+    private int metadataThreadPoolNums = 3;
+    private int metadataThreadPoolQueueCapacity = 1000;
+
+    private int transactionHeartbeatThreadPoolNums = 20;
+    private int transactionHeartbeatThreadPoolQueueCapacity = 200;
+    private int transactionHeartbeatPeriodSecond = 20;
+    private int transactionHeartbeatBatchNum = 100;
+    private long transactionDataExpireScanPeriodMillis = Duration.ofSeconds(10).toMillis();
+    private long transactionDataMaxWaitClearMillis = Duration.ofSeconds(30).toMillis();
+    private long transactionDataExpireMillis = Duration.ofSeconds(30).toMillis();
+    private int transactionDataMaxNum = 15;
+
+    private long longPollingReserveTimeInMillis = 100;
+
+    private long invisibleTimeMillisWhenClear = 1000L;
+    private boolean enableProxyAutoRenew = true;
+    private long renewAheadTimeMillis = TimeUnit.SECONDS.toMillis(10);
+    private long renewSliceTimeMillis = TimeUnit.SECONDS.toMillis(60);
+    private long renewMaxTimeMillis = TimeUnit.HOURS.toMillis(3);
+    private long renewSchedulePeriodMillis = TimeUnit.SECONDS.toMillis(5);
+
+    private boolean enableACL = false;
+
+    private boolean useDelayLevel = true;
+    private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h";
+    private transient Map<Integer /* level */, Long/* delay timeMillis */> delayLevelTable = new ConcurrentHashMap<>();
+
+    private int metricCollectorMode = MetricCollectorMode.OFF.getOrdinal();
+    // Example address: 127.0.0.1:1234
+    private String metricCollectorAddress = "";
+
+    @Override
+    public void initData() {
+        parseDelayLevel();
+    }
+
+    public int computeDelayLevel(long timeMillis) {
+        long intervalMillis = timeMillis - System.currentTimeMillis();
+        List<Map.Entry<Integer, Long>> sortedLevels = delayLevelTable.entrySet().stream().sorted(Comparator.comparingLong(Map.Entry::getValue)).collect(Collectors.toList());
+        for (Map.Entry<Integer, Long> entry : sortedLevels) {
+            if (entry.getValue() > intervalMillis) {
+                return entry.getKey();
+            }
+        }
+        return sortedLevels.get(sortedLevels.size() - 1).getKey();
+    }
+
+    public void parseDelayLevel() {
+        this.delayLevelTable = new ConcurrentHashMap<>();
+        Map<String, Long> timeUnitTable = new HashMap<>();
+        timeUnitTable.put("s", 1000L);
+        timeUnitTable.put("m", 1000L * 60);
+        timeUnitTable.put("h", 1000L * 60 * 60);
+        timeUnitTable.put("d", 1000L * 60 * 60 * 24);
+
+        String levelString = this.getMessageDelayLevel();
+        try {
+            String[] levelArray = levelString.split(" ");
+            for (int i = 0; i < levelArray.length; i++) {
+                String value = levelArray[i];
+                String ch = value.substring(value.length() - 1);
+                Long tu = timeUnitTable.get(ch);
+
+                int level = i + 1;
+                long num = Long.parseLong(value.substring(0, value.length() - 1));
+                long delayTimeMillis = tu * num;
+                this.delayLevelTable.put(level, delayTimeMillis);
+            }
+        } catch (Exception e) {
+            log.error("parse delay level failed. messageDelayLevel:{}", messageDelayLevel, e);
+        }
+    }
+
+    public String getRocketMQClusterName() {
+        return rocketMQClusterName;
+    }
+
+    public void setRocketMQClusterName(String rocketMQClusterName) {
+        this.rocketMQClusterName = rocketMQClusterName;
+    }
+
+    public boolean isEnablePrintJstack() {
+        return enablePrintJstack;
+    }
+
+    public void setEnablePrintJstack(boolean enablePrintJstack) {
+        this.enablePrintJstack = enablePrintJstack;
+    }
+
+    public long getPrintJstackInMillis() {
+        return printJstackInMillis;
+    }
+
+    public void setPrintJstackInMillis(long printJstackInMillis) {
+        this.printJstackInMillis = printJstackInMillis;
+    }
+
+    public long getPrintThreadPoolStatusInMillis() {
+        return printThreadPoolStatusInMillis;
+    }
+
+    public void setPrintThreadPoolStatusInMillis(long printThreadPoolStatusInMillis) {
+        this.printThreadPoolStatusInMillis = printThreadPoolStatusInMillis;
+    }
+
+    public String getNameSrvAddr() {
+        return nameSrvAddr;
+    }
+
+    public void setNameSrvAddr(String nameSrvAddr) {
+        this.nameSrvAddr = nameSrvAddr;
+    }
+
+    public String getNameSrvDomain() {
+        return nameSrvDomain;
+    }
+
+    public void setNameSrvDomain(String nameSrvDomain) {
+        this.nameSrvDomain = nameSrvDomain;
+    }
+
+    public String getNameSrvDomainSubgroup() {
+        return nameSrvDomainSubgroup;
+    }
+
+    public void setNameSrvDomainSubgroup(String nameSrvDomainSubgroup) {
+        this.nameSrvDomainSubgroup = nameSrvDomainSubgroup;
+    }
+
+    public String getProxyMode() {
+        return proxyMode;
+    }
+
+    public void setProxyMode(String proxyMode) {
+        this.proxyMode = proxyMode;
+    }
+
+    public Integer getGrpcServerPort() {
+        return grpcServerPort;
+    }
+
+    public void setGrpcServerPort(Integer grpcServerPort) {
+        this.grpcServerPort = grpcServerPort;
+    }
+
+    public boolean isGrpcTlsTestModeEnable() {
+        return grpcTlsTestModeEnable;
+    }
+
+    public void setGrpcTlsTestModeEnable(boolean grpcTlsTestModeEnable) {
+        this.grpcTlsTestModeEnable = grpcTlsTestModeEnable;
+    }
+
+    public String getGrpcTlsKeyPath() {
+        return grpcTlsKeyPath;
+    }
+
+    public void setGrpcTlsKeyPath(String grpcTlsKeyPath) {
+        this.grpcTlsKeyPath = grpcTlsKeyPath;
+    }
+
+    public String getGrpcTlsCertPath() {
+        return grpcTlsCertPath;
+    }
+
+    public void setGrpcTlsCertPath(String grpcTlsCertPath) {
+        this.grpcTlsCertPath = grpcTlsCertPath;
+    }
+
+    public int getGrpcBossLoopNum() {
+        return grpcBossLoopNum;
+    }
+
+    public void setGrpcBossLoopNum(int grpcBossLoopNum) {
+        this.grpcBossLoopNum = grpcBossLoopNum;
+    }
+
+    public int getGrpcWorkerLoopNum() {
+        return grpcWorkerLoopNum;
+    }
+
+    public void setGrpcWorkerLoopNum(int grpcWorkerLoopNum) {
+        this.grpcWorkerLoopNum = grpcWorkerLoopNum;
+    }
+
+    public boolean isEnableGrpcEpoll() {
+        return enableGrpcEpoll;
+    }
+
+    public void setEnableGrpcEpoll(boolean enableGrpcEpoll) {
+        this.enableGrpcEpoll = enableGrpcEpoll;
+    }
+
+    public int getGrpcThreadPoolNums() {
+        return grpcThreadPoolNums;
+    }
+
+    public void setGrpcThreadPoolNums(int grpcThreadPoolNums) {
+        this.grpcThreadPoolNums = grpcThreadPoolNums;
+    }
+
+    public int getGrpcThreadPoolQueueCapacity() {
+        return grpcThreadPoolQueueCapacity;
+    }
+
+    public void setGrpcThreadPoolQueueCapacity(int grpcThreadPoolQueueCapacity) {
+        this.grpcThreadPoolQueueCapacity = grpcThreadPoolQueueCapacity;
+    }
+
+    public String getBrokerConfigPath() {
+        return brokerConfigPath;
+    }
+
+    public void setBrokerConfigPath(String brokerConfigPath) {
+        this.brokerConfigPath = brokerConfigPath;
+    }
+
+    public int getGrpcMaxInboundMessageSize() {
+        return grpcMaxInboundMessageSize;
+    }
+
+    public void setGrpcMaxInboundMessageSize(int grpcMaxInboundMessageSize) {
+        this.grpcMaxInboundMessageSize = grpcMaxInboundMessageSize;
+    }
+
+    public int getMaxMessageSize() {
+        return maxMessageSize;
+    }
+
+    public void setMaxMessageSize(int maxMessageSize) {
+        this.maxMessageSize = maxMessageSize;
+    }
+
+    public int getMaxUserPropertySize() {
+        return maxUserPropertySize;
+    }
+
+    public void setMaxUserPropertySize(int maxUserPropertySize) {
+        this.maxUserPropertySize = maxUserPropertySize;
+    }
+
+    public int getUserPropertyMaxNum() {
+        return userPropertyMaxNum;
+    }
+
+    public void setUserPropertyMaxNum(int userPropertyMaxNum) {
+        this.userPropertyMaxNum = userPropertyMaxNum;
+    }
+
+    public int getMaxMessageGroupSize() {
+        return maxMessageGroupSize;
+    }
+
+    public void setMaxMessageGroupSize(int maxMessageGroupSize) {
+        this.maxMessageGroupSize = maxMessageGroupSize;
+    }
+
+    public long getMinInvisibleTimeMillsForRecv() {
+        return minInvisibleTimeMillsForRecv;
+    }
+
+    public void setMinInvisibleTimeMillsForRecv(long minInvisibleTimeMillsForRecv) {
+        this.minInvisibleTimeMillsForRecv = minInvisibleTimeMillsForRecv;
+    }
+
+    public long getMaxInvisibleTimeMills() {
+        return maxInvisibleTimeMills;
+    }
+
+    public void setMaxInvisibleTimeMills(long maxInvisibleTimeMills) {
+        this.maxInvisibleTimeMills = maxInvisibleTimeMills;
+    }
+
+    public long getMaxDelayTimeMills() {
+        return maxDelayTimeMills;
+    }
+
+    public void setMaxDelayTimeMills(long maxDelayTimeMills) {
+        this.maxDelayTimeMills = maxDelayTimeMills;
+    }
+
+    public long getMaxTransactionRecoverySecond() {
+        return maxTransactionRecoverySecond;
+    }
+
+    public void setMaxTransactionRecoverySecond(long maxTransactionRecoverySecond) {
+        this.maxTransactionRecoverySecond = maxTransactionRecoverySecond;
+    }
+
+    public int getGrpcClientProducerMaxAttempts() {
+        return grpcClientProducerMaxAttempts;
+    }
+
+    public void setGrpcClientProducerMaxAttempts(int grpcClientProducerMaxAttempts) {
+        this.grpcClientProducerMaxAttempts = grpcClientProducerMaxAttempts;
+    }
+
+    public long getGrpcClientProducerBackoffInitialMillis() {
+        return grpcClientProducerBackoffInitialMillis;
+    }
+
+    public void setGrpcClientProducerBackoffInitialMillis(long grpcClientProducerBackoffInitialMillis) {
+        this.grpcClientProducerBackoffInitialMillis = grpcClientProducerBackoffInitialMillis;
+    }
+
+    public long getGrpcClientProducerBackoffMaxMillis() {
+        return grpcClientProducerBackoffMaxMillis;
+    }
+
+    public void setGrpcClientProducerBackoffMaxMillis(long grpcClientProducerBackoffMaxMillis) {
+        this.grpcClientProducerBackoffMaxMillis = grpcClientProducerBackoffMaxMillis;
+    }
+
+    public int getGrpcClientProducerBackoffMultiplier() {
+        return grpcClientProducerBackoffMultiplier;
+    }
+
+    public void setGrpcClientProducerBackoffMultiplier(int grpcClientProducerBackoffMultiplier) {
+        this.grpcClientProducerBackoffMultiplier = grpcClientProducerBackoffMultiplier;
+    }
+
+    public long getGrpcClientConsumerLongPollingTimeoutMillis() {
+        return grpcClientConsumerLongPollingTimeoutMillis;
+    }
+
+    public void setGrpcClientConsumerLongPollingTimeoutMillis(long grpcClientConsumerLongPollingTimeoutMillis) {
+        this.grpcClientConsumerLongPollingTimeoutMillis = grpcClientConsumerLongPollingTimeoutMillis;
+    }
+
+    public int getGrpcClientConsumerLongPollingBatchSize() {
+        return grpcClientConsumerLongPollingBatchSize;
+    }
+
+    public void setGrpcClientConsumerLongPollingBatchSize(int grpcClientConsumerLongPollingBatchSize) {
+        this.grpcClientConsumerLongPollingBatchSize = grpcClientConsumerLongPollingBatchSize;
+    }
+
+    public int getChannelExpiredInSeconds() {
+        return channelExpiredInSeconds;
+    }
+
+    public void setChannelExpiredInSeconds(int channelExpiredInSeconds) {
+        this.channelExpiredInSeconds = channelExpiredInSeconds;
+    }
+
+    public int getContextExpiredInSeconds() {
+        return contextExpiredInSeconds;
+    }
+
+    public void setContextExpiredInSeconds(int contextExpiredInSeconds) {
+        this.contextExpiredInSeconds = contextExpiredInSeconds;
+    }
+
+    public int getRocketmqMQClientNum() {
+        return rocketmqMQClientNum;
+    }
+
+    public void setRocketmqMQClientNum(int rocketmqMQClientNum) {
+        this.rocketmqMQClientNum = rocketmqMQClientNum;
+    }
+
+    public long getGrpcProxyRelayRequestTimeoutInSeconds() {
+        return grpcProxyRelayRequestTimeoutInSeconds;
+    }
+
+    public void setGrpcProxyRelayRequestTimeoutInSeconds(long grpcProxyRelayRequestTimeoutInSeconds) {
+        this.grpcProxyRelayRequestTimeoutInSeconds = grpcProxyRelayRequestTimeoutInSeconds;
+    }
+
+    public int getGrpcProducerThreadPoolNums() {
+        return grpcProducerThreadPoolNums;
+    }
+
+    public void setGrpcProducerThreadPoolNums(int grpcProducerThreadPoolNums) {
+        this.grpcProducerThreadPoolNums = grpcProducerThreadPoolNums;
+    }
+
+    public int getGrpcProducerThreadQueueCapacity() {
+        return grpcProducerThreadQueueCapacity;
+    }
+
+    public void setGrpcProducerThreadQueueCapacity(int grpcProducerThreadQueueCapacity) {
+        this.grpcProducerThreadQueueCapacity = grpcProducerThreadQueueCapacity;
+    }
+
+    public int getGrpcConsumerThreadPoolNums() {
+        return grpcConsumerThreadPoolNums;
+    }
+
+    public void setGrpcConsumerThreadPoolNums(int grpcConsumerThreadPoolNums) {
+        this.grpcConsumerThreadPoolNums = grpcConsumerThreadPoolNums;
+    }
+
+    public int getGrpcConsumerThreadQueueCapacity() {
+        return grpcConsumerThreadQueueCapacity;
+    }
+
+    public void setGrpcConsumerThreadQueueCapacity(int grpcConsumerThreadQueueCapacity) {
+        this.grpcConsumerThreadQueueCapacity = grpcConsumerThreadQueueCapacity;
+    }
+
+    public int getGrpcRouteThreadPoolNums() {
+        return grpcRouteThreadPoolNums;
+    }
+
+    public void setGrpcRouteThreadPoolNums(int grpcRouteThreadPoolNums) {
+        this.grpcRouteThreadPoolNums = grpcRouteThreadPoolNums;
+    }
+
+    public int getGrpcRouteThreadQueueCapacity() {
+        return grpcRouteThreadQueueCapacity;
+    }
+
+    public void setGrpcRouteThreadQueueCapacity(int grpcRouteThreadQueueCapacity) {
+        this.grpcRouteThreadQueueCapacity = grpcRouteThreadQueueCapacity;
+    }
+
+    public int getGrpcClientManagerThreadPoolNums() {
+        return grpcClientManagerThreadPoolNums;
+    }
+
+    public void setGrpcClientManagerThreadPoolNums(int grpcClientManagerThreadPoolNums) {
+        this.grpcClientManagerThreadPoolNums = grpcClientManagerThreadPoolNums;
+    }
+
+    public int getGrpcClientManagerThreadQueueCapacity() {
+        return grpcClientManagerThreadQueueCapacity;
+    }
+
+    public void setGrpcClientManagerThreadQueueCapacity(int grpcClientManagerThreadQueueCapacity) {
+        this.grpcClientManagerThreadQueueCapacity = grpcClientManagerThreadQueueCapacity;
+    }
+
+    public int getGrpcTransactionThreadPoolNums() {
+        return grpcTransactionThreadPoolNums;
+    }
+
+    public void setGrpcTransactionThreadPoolNums(int grpcTransactionThreadPoolNums) {
+        this.grpcTransactionThreadPoolNums = grpcTransactionThreadPoolNums;
+    }
+
+    public int getGrpcTransactionThreadQueueCapacity() {
+        return grpcTransactionThreadQueueCapacity;
+    }
+
+    public void setGrpcTransactionThreadQueueCapacity(int grpcTransactionThreadQueueCapacity) {
+        this.grpcTransactionThreadQueueCapacity = grpcTransactionThreadQueueCapacity;
+    }
+
+    public int getProducerProcessorThreadPoolNums() {
+        return producerProcessorThreadPoolNums;
+    }
+
+    public void setProducerProcessorThreadPoolNums(int producerProcessorThreadPoolNums) {
+        this.producerProcessorThreadPoolNums = producerProcessorThreadPoolNums;
+    }
+
+    public int getProducerProcessorThreadPoolQueueCapacity() {
+        return producerProcessorThreadPoolQueueCapacity;
+    }
+
+    public void setProducerProcessorThreadPoolQueueCapacity(int producerProcessorThreadPoolQueueCapacity) {
+        this.producerProcessorThreadPoolQueueCapacity = producerProcessorThreadPoolQueueCapacity;
+    }
+
+    public int getConsumerProcessorThreadPoolNums() {
+        return consumerProcessorThreadPoolNums;
+    }
+
+    public void setConsumerProcessorThreadPoolNums(int consumerProcessorThreadPoolNums) {
+        this.consumerProcessorThreadPoolNums = consumerProcessorThreadPoolNums;
+    }
+
+    public int getConsumerProcessorThreadPoolQueueCapacity() {
+        return consumerProcessorThreadPoolQueueCapacity;
+    }
+
+    public void setConsumerProcessorThreadPoolQueueCapacity(int consumerProcessorThreadPoolQueueCapacity) {
+        this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity;
+    }
+
+    public int getTopicRouteServiceCacheExpiredInSeconds() {
+        return topicRouteServiceCacheExpiredInSeconds;
+    }
+
+    public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) {
+        this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds;
+    }
+
+    public int getTopicRouteServiceCacheMaxNum() {
+        return topicRouteServiceCacheMaxNum;
+    }
+
+    public void setTopicRouteServiceCacheMaxNum(int topicRouteServiceCacheMaxNum) {
+        this.topicRouteServiceCacheMaxNum = topicRouteServiceCacheMaxNum;
+    }
+
+    public int getTopicRouteServiceThreadPoolNums() {
+        return topicRouteServiceThreadPoolNums;
+    }
+
+    public void setTopicRouteServiceThreadPoolNums(int topicRouteServiceThreadPoolNums) {
+        this.topicRouteServiceThreadPoolNums = topicRouteServiceThreadPoolNums;
+    }
+
+    public int getTopicRouteServiceThreadPoolQueueCapacity() {
+        return topicRouteServiceThreadPoolQueueCapacity;
+    }
+
+    public void setTopicRouteServiceThreadPoolQueueCapacity(int topicRouteServiceThreadPoolQueueCapacity) {
+        this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity;
+    }
+
+    public int getTopicConfigCacheExpiredInSeconds() {
+        return topicConfigCacheExpiredInSeconds;
+    }
+
+    public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) {
+        this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds;
+    }
+
+    public int getTopicConfigCacheMaxNum() {
+        return topicConfigCacheMaxNum;
+    }
+
+    public void setTopicConfigCacheMaxNum(int topicConfigCacheMaxNum) {
+        this.topicConfigCacheMaxNum = topicConfigCacheMaxNum;
+    }
+
+    public int getSubscriptionGroupConfigCacheExpiredInSeconds() {
+        return subscriptionGroupConfigCacheExpiredInSeconds;
+    }
+
+    public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) {
+        this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds;
+    }
+
+    public int getSubscriptionGroupConfigCacheMaxNum() {
+        return subscriptionGroupConfigCacheMaxNum;
+    }
+
+    public void setSubscriptionGroupConfigCacheMaxNum(int subscriptionGroupConfigCacheMaxNum) {
+        this.subscriptionGroupConfigCacheMaxNum = subscriptionGroupConfigCacheMaxNum;
+    }
+
+    public int getMetadataThreadPoolNums() {
+        return metadataThreadPoolNums;
+    }
+
+    public void setMetadataThreadPoolNums(int metadataThreadPoolNums) {
+        this.metadataThreadPoolNums = metadataThreadPoolNums;
+    }
+
+    public int getMetadataThreadPoolQueueCapacity() {
+        return metadataThreadPoolQueueCapacity;
+    }
+
+    public void setMetadataThreadPoolQueueCapacity(int metadataThreadPoolQueueCapacity) {
+        this.metadataThreadPoolQueueCapacity = metadataThreadPoolQueueCapacity;
+    }
+
+    public int getTransactionHeartbeatThreadPoolNums() {
+        return transactionHeartbeatThreadPoolNums;
+    }
+
+    public void setTransactionHeartbeatThreadPoolNums(int transactionHeartbeatThreadPoolNums) {
+        this.transactionHeartbeatThreadPoolNums = transactionHeartbeatThreadPoolNums;
+    }
+
+    public int getTransactionHeartbeatThreadPoolQueueCapacity() {
+        return transactionHeartbeatThreadPoolQueueCapacity;
+    }
+
+    public void setTransactionHeartbeatThreadPoolQueueCapacity(int transactionHeartbeatThreadPoolQueueCapacity) {
+        this.transactionHeartbeatThreadPoolQueueCapacity = transactionHeartbeatThreadPoolQueueCapacity;
+    }
+
+    public int getTransactionHeartbeatPeriodSecond() {
+        return transactionHeartbeatPeriodSecond;
+    }
+
+    public void setTransactionHeartbeatPeriodSecond(int transactionHeartbeatPeriodSecond) {
+        this.transactionHeartbeatPeriodSecond = transactionHeartbeatPeriodSecond;
+    }
+
+    public int getTransactionHeartbeatBatchNum() {
+        return transactionHeartbeatBatchNum;
+    }
+
+    public void setTransactionHeartbeatBatchNum(int transactionHeartbeatBatchNum) {
+        this.transactionHeartbeatBatchNum = transactionHeartbeatBatchNum;
+    }
+
+    public long getTransactionDataExpireScanPeriodMillis() {
+        return transactionDataExpireScanPeriodMillis;
+    }
+
+    public void setTransactionDataExpireScanPeriodMillis(long transactionDataExpireScanPeriodMillis) {
+        this.transactionDataExpireScanPeriodMillis = transactionDataExpireScanPeriodMillis;
+    }
+
+    public long getTransactionDataMaxWaitClearMillis() {
+        return transactionDataMaxWaitClearMillis;
+    }
+
+    public void setTransactionDataMaxWaitClearMillis(long transactionDataMaxWaitClearMillis) {
+        this.transactionDataMaxWaitClearMillis = transactionDataMaxWaitClearMillis;
+    }
+
+    public long getTransactionDataExpireMillis() {
+        return transactionDataExpireMillis;
+    }
+
+    public void setTransactionDataExpireMillis(long transactionDataExpireMillis) {
+        this.transactionDataExpireMillis = transactionDataExpireMillis;
+    }
+
+    public int getTransactionDataMaxNum() {
+        return transactionDataMaxNum;
+    }
+
+    public void setTransactionDataMaxNum(int transactionDataMaxNum) {
+        this.transactionDataMaxNum = transactionDataMaxNum;
+    }
+
+    public long getLongPollingReserveTimeInMillis() {
+        return longPollingReserveTimeInMillis;
+    }
+
+    public void setLongPollingReserveTimeInMillis(long longPollingReserveTimeInMillis) {
+        this.longPollingReserveTimeInMillis = longPollingReserveTimeInMillis;
+    }
+
+    public boolean isEnableACL() {
+        return enableACL;
+    }
+
+    public void setEnableACL(boolean enableACL) {
+        this.enableACL = enableACL;
+    }
+
+    public boolean isEnableTopicMessageTypeCheck() {
+        return enableTopicMessageTypeCheck;
+    }
+
+    public void setEnableTopicMessageTypeCheck(boolean enableTopicMessageTypeCheck) {
+        this.enableTopicMessageTypeCheck = enableTopicMessageTypeCheck;
+    }
+
+    public long getInvisibleTimeMillisWhenClear() {
+        return invisibleTimeMillisWhenClear;
+    }
+
+    public void setInvisibleTimeMillisWhenClear(long invisibleTimeMillisWhenClear) {
+        this.invisibleTimeMillisWhenClear = invisibleTimeMillisWhenClear;
+    }
+
+    public boolean isEnableProxyAutoRenew() {
+        return enableProxyAutoRenew;
+    }
+
+    public void setEnableProxyAutoRenew(boolean enableProxyAutoRenew) {
+        this.enableProxyAutoRenew = enableProxyAutoRenew;
+    }
+
+    public long getRenewAheadTimeMillis() {
+        return renewAheadTimeMillis;
+    }
+
+    public void setRenewAheadTimeMillis(long renewAheadTimeMillis) {
+        this.renewAheadTimeMillis = renewAheadTimeMillis;
+    }
+
+    public long getRenewSliceTimeMillis() {
+        return renewSliceTimeMillis;
+    }
+
+    public void setRenewSliceTimeMillis(long renewSliceTimeMillis) {
+        this.renewSliceTimeMillis = renewSliceTimeMillis;
+    }
+
+    public long getRenewMaxTimeMillis() {
+        return renewMaxTimeMillis;
+    }
+
+    public void setRenewMaxTimeMillis(long renewMaxTimeMillis) {
+        this.renewMaxTimeMillis = renewMaxTimeMillis;
+    }
+
+    public long getRenewSchedulePeriodMillis() {
+        return renewSchedulePeriodMillis;
+    }
+
+    public void setRenewSchedulePeriodMillis(long renewSchedulePeriodMillis) {
+        this.renewSchedulePeriodMillis = renewSchedulePeriodMillis;
+    }
+
+    public int getMetricCollectorMode() {
+        return metricCollectorMode;
+    }
+
+    public void setMetricCollectorMode(int metricCollectorMode) {
+        this.metricCollectorMode = metricCollectorMode;
+    }
+
+    public String getMetricCollectorAddress() {
+        return metricCollectorAddress;
+    }
+
+    public void setMetricCollectorAddress(String metricCollectorAddress) {
+        this.metricCollectorAddress = metricCollectorAddress;
+    }
+
+    public boolean isUseDelayLevel() {
+        return useDelayLevel;
+    }
+
+    public void setUseDelayLevel(boolean useDelayLevel) {
+        this.useDelayLevel = useDelayLevel;
+    }
+
+    public String getMessageDelayLevel() {
+        return messageDelayLevel;
+    }
+
+    public void setMessageDelayLevel(String messageDelayLevel) {
+        this.messageDelayLevel = messageDelayLevel;
+    }
+
+    public Map<Integer, Long> getDelayLevelTable() {
+        return delayLevelTable;
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.proxy.grpc;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.common.StartAndShutdown;
+
+public class GrpcServer implements StartAndShutdown {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+
+    private final io.grpc.Server server;
+
+    protected GrpcServer(io.grpc.Server server) {
+        this.server = server;
+    }
+
+    public void start() throws Exception {
+        this.server.start();
+        log.info("grpc server start successfully.");
+    }
+
+    public void shutdown() {
+        try {
+            this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS);
+            log.info("grpc server shutdown successfully.");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java

@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.proxy.grpc;
+
+import io.grpc.BindableService;
+import io.grpc.ServerInterceptor;
+import io.grpc.ServerServiceDefinition;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder;
+import io.grpc.netty.shaded.io.netty.channel.epoll.EpollEventLoopGroup;
+import io.grpc.netty.shaded.io.netty.channel.epoll.EpollServerSocketChannel;
+import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup;
+import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioServerSocketChannel;
+import io.grpc.netty.shaded.io.netty.handler.ssl.ClientAuth;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.SelfSignedCertificate;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.cert.CertificateException;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+import javax.net.ssl.SSLException;
+import org.apache.rocketmq.acl.AccessValidator;
+import org.apache.rocketmq.common.constant.LoggerName;
+import org.apache.rocketmq.common.utils.ServiceProvider;
+import org.apache.rocketmq.logging.InternalLogger;
+import org.apache.rocketmq.logging.InternalLoggerFactory;
+import org.apache.rocketmq.proxy.config.ConfigurationManager;
+import org.apache.rocketmq.proxy.config.ProxyConfig;
+import org.apache.rocketmq.proxy.grpc.interceptor.AuthenticationInterceptor;
+import org.apache.rocketmq.proxy.grpc.interceptor.ContextInterceptor;
+import org.apache.rocketmq.proxy.grpc.interceptor.GlobalExceptionInterceptor;
+import org.apache.rocketmq.proxy.grpc.interceptor.HeaderInterceptor;
+
+public class GrpcServerBuilder {
+    private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME);
+    protected NettyServerBuilder serverBuilder;
+
+    public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) {
+        return new GrpcServerBuilder(executor, port);
+    }
+
+    protected GrpcServerBuilder(ThreadPoolExecutor executor, int port) {
+        serverBuilder = NettyServerBuilder.forPort(port);
+
+        try {
+            configSslContext(serverBuilder);
+        } catch (Exception e) {
+            log.error("grpc tls set failed. msg: {}, e:", e.getMessage(), e);
+            throw new RuntimeException("grpc tls set failed: " + e.getMessage());
+        }
+
+        // build server
+        int bossLoopNum = ConfigurationManager.getProxyConfig().getGrpcBossLoopNum();
+        int workerLoopNum = ConfigurationManager.getProxyConfig().getGrpcWorkerLoopNum();
+        int maxInboundMessageSize = ConfigurationManager.getProxyConfig().getGrpcMaxInboundMessageSize();
+
+        if (ConfigurationManager.getProxyConfig().isEnableGrpcEpoll()) {
+            serverBuilder.maxInboundMessageSize(maxInboundMessageSize)
+                .bossEventLoopGroup(new EpollEventLoopGroup(bossLoopNum))
+                .workerEventLoopGroup(new EpollEventLoopGroup(workerLoopNum))
+                .channelType(EpollServerSocketChannel.class)
+                .executor(executor);
+        } else {
+            serverBuilder.maxInboundMessageSize(maxInboundMessageSize)
+                .bossEventLoopGroup(new NioEventLoopGroup(bossLoopNum))
+                .workerEventLoopGroup(new NioEventLoopGroup(workerLoopNum))
+                .channelType(NioServerSocketChannel.class)
+                .executor(executor);
+        }
+
+        log.info(
+            "grpc server has built. port: {}, tlsKeyPath: {}, tlsCertPath: {}, threadPool: {}, queueCapacity: {}, "
+                + "boosLoop: {}, workerLoop: {}, maxInboundMessageSize: {}",
+            port, bossLoopNum, workerLoopNum, maxInboundMessageSize);
+    }
+
+    public GrpcServerBuilder addService(BindableService service) {
+        this.serverBuilder.addService(service);
+        return this;
+    }
+
+    public GrpcServerBuilder addService(ServerServiceDefinition service) {
+        this.serverBuilder.addService(service);
+        return this;
+    }
+
+    public GrpcServerBuilder appendInterceptor(ServerInterceptor interceptor) {
+        this.serverBuilder.intercept(interceptor);
+        return this;
+    }
+
+    public GrpcServer build() {
+        return new GrpcServer(this.serverBuilder.build());
+    }
+
+    protected void configSslContext(NettyServerBuilder serverBuilder) throws SSLException, CertificateException {
+        if (null == serverBuilder) {
+            return;
+        }
+        ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig();
+        boolean tlsTestModeEnable = proxyConfig.isGrpcTlsTestModeEnable();
+        if (tlsTestModeEnable) {
+            SelfSignedCertificate selfSignedCertificate = new SelfSignedCertificate();
+            serverBuilder.sslContext(GrpcSslContexts.forServer(selfSignedCertificate.certificate(), selfSignedCertificate.privateKey())
+                .trustManager(InsecureTrustManagerFactory.INSTANCE)
+                .clientAuth(ClientAuth.NONE)
+                .build());
+            return;
+        }
+
+        String tlsKeyPath = ConfigurationManager.getProxyConfig().getGrpcTlsKeyPath();
+        String tlsCertPath = ConfigurationManager.getProxyConfig().getGrpcTlsCertPath();
+        try (InputStream serverKeyInputStream = Files.newInputStream(Paths.get(tlsKeyPath));
+             InputStream serverCertificateStream = Files.newInputStream(Paths.get(tlsCertPath))) {
+            serverBuilder.sslContext(GrpcSslContexts.forServer(serverCertificateStream, serverKeyInputStream)
+                .trustManager(InsecureTrustManagerFactory.INSTANCE)
+                .clientAuth(ClientAuth.NONE)
+                .build());
+            log.info("TLS configured OK");
+        } catch (IOException e) {
+            log.error("Failed to load Server key/certificate", e);
+        }
+    }
+
+    public GrpcServerBuilder configInterceptor() {
+        // grpc interceptors, including acl, logging etc.
+        List<AccessValidator> accessValidators = ServiceProvider.load(ServiceProvider.ACL_VALIDATOR_ID, AccessValidator.class);
+        if (!accessValidators.isEmpty()) {
+            this.serverBuilder.intercept(new AuthenticationInterceptor(accessValidators));
+        }
+
+        this.serverBuilder
+            .intercept(new GlobalExceptionInterceptor())
+            .intercept(new ContextInterceptor())
+            .intercept(new HeaderInterceptor());
+
+        return this;
+    }
+
+}